def execute(
self,
command=None,
container_id=None,
sudo=False,
stream=False,
singularity_options=None,
):
"""execute a command to a container instance based on container_id
Parameters
==========
container_id: the container_id to delete
command: the command to execute to the container
singularity_options: a list of options to provide to the singularity client
sudo: whether to issue the command with sudo (or not)
a container started with sudo will belong to the root user
If started by a user, the user needs to control deleting it
stream: if True, return an iterate to iterate over results of exec.
default is False, will return full output as string.
Returns
=======
return_code: the return code from the delete command. 0 indicates a
successful delete, 255 indicates not.
"""
sudo = self._get_sudo(sudo)
container_id = self.get_container_id(container_id)
# singularity oci delete
cmd = self._init_command("exec", singularity_options)
# Add the container_id
cmd.append(container_id)
if command is not None:
if not isinstance(command, list):
command = [command]
cmd = cmd + command
# Execute the command, return response to user
if stream:
return stream_command(cmd, sudo=sudo)
return self._run_command(cmd, sudo=sudo, quiet=True)
def pull(self,
image=None,
name=None,
pull_folder='',
ext="simg",
force=False,
capture=False,
name_by_commit=False,
name_by_hash=False,
stream=False):
'''pull will pull a singularity hub or Docker image
Parameters
==========
image: the complete image uri. If not provided, the client loaded is used
pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to
user specified location instead.
Docker and Singularity Hub Naming
---------------------------------
name: a custom name to use, to override default
ext: if no name specified, the default extension to use.
'''
from spython.utils import check_install
check_install()
cmd = self._init_command('pull')
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# If it's still None, no go!
if image is None:
bot.exit('You must provide an image uri, or use client.load() first.')
# Singularity Only supports shub and Docker pull
if not re.search('^(shub|docker)://', image):
bot.exit("pull only valid for docker and shub. Use sregistry client.")
# Did the user ask for a custom pull folder?
if pull_folder:
self.setenv('SINGULARITY_PULLFOLDER', pull_folder)
# If we still don't have a custom name, base off of image uri.
# Determine how to tell client to name the image, preference to hash
if name_by_hash is True:
cmd.append('--hash')
elif name_by_commit is True:
cmd.append('--commit')
elif name is None:
name = self._get_filename(image, ext)
# Only add name if we aren't naming by hash or commit
if not name_by_commit and not name_by_hash:
cmd = cmd + ["--name", name]
if force is True:
cmd = cmd + ["--force"]
cmd.append(image)
bot.info(' '.join(cmd))
# If name is still None, make empty string
if name is None:
name = ''
final_image = os.path.join(pull_folder, name)
# Option 1: For hash or commit, need return value to get final_image
if name_by_commit or name_by_hash:
# Set pull to temporary location
tmp_folder = tempfile.mkdtemp()
self.setenv('SINGULARITY_PULLFOLDER', tmp_folder)
self._run_command(cmd, capture=capture)
try:
tmp_image = os.path.join(tmp_folder, os.listdir(tmp_folder)[0])
final_image = os.path.join(pull_folder,
os.path.basename(tmp_image))
shutil.move(tmp_image, final_image)
shutil.rmtree(tmp_folder)
except:
bot.error('Issue pulling image with commit or hash, try without?')
# Option 2: Streaming we just run to show user
elif stream is False:
self._run_command(cmd, capture=capture)
# Option 3: A custom name we can predict (not commit/hash) and can also show
else:
return final_image, stream_command(cmd, sudo=False)
if os.path.exists(final_image):
bot.info(final_image)
return final_image
def run(self,
image=None,
args=None,
app=None,
sudo=False,
writable=False,
contain=False,
bind=None,
stream=False):
'''
run will run the container, with or withour arguments (which
should be provided in a list)
Parameters
==========
image: full path to singularity image
args: args to include with the run
app: if not None, execute a command in context of an app
writable: This option makes the file system accessible as read/write
contain: This option disables the automatic sharing of writable
filesystems on your host
bind: list or single string of bind paths.
This option allows you to map directories on your host system to
directories within your container using bind mounts
stream: if True, return <generator> for the user to run
'''
self.check_install()
cmd = self._init_command('run')
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# Does the user want to use bind paths option?
if bind is not None:
cmd += self._generate_bind_list(bind)
# Does the user want to run an app?
if app is not None:
cmd = cmd + ['--app', app]
cmd = cmd + [image]
# Conditions for needing sudo
if writable is True:
sudo = True
if args is not None:
if not isinstance(args, list):
args = args.split(' ')
cmd = cmd + args
if stream is False:
result = self._run_command(cmd, sudo=sudo)
else:
return stream_command(cmd, sudo=sudo)
result = result.strip('\n')
try:
result = json.loads(result)
except:
pass
return result
def run(
self,
image=None,
args=None,
app=None,
sudo=False,
writable=False,
contain=False,
bind=None,
stream=False,
nv=False,
options=None,
return_result=False,
):
"""
run will run the container, with or withour arguments (which
should be provided in a list)
Parameters
==========
image: full path to singularity image
args: args to include with the run
app: if not None, execute a command in context of an app
writable: This option makes the file system accessible as read/write
options: an optional list of options to provide to run.
contain: This option disables the automatic sharing of writable
filesystems on your host
bind: list or single string of bind paths.
This option allows you to map directories on your host system to
directories within your container using bind mounts
stream: if True, return <generator> for the user to run
nv: if True, load Nvidia Drivers in runtime (default False)
return_result: if True, return entire json object with return code
and message result (default is False)
"""
from spython.utils import check_install
check_install()
cmd = self._init_command("run")
# nv option leverages any GPU cards
if nv:
cmd += ["--nv"]
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# If an instance is provided, grab it's name
if isinstance(image, self.instance):
image = image.get_uri()
# If image is still None, not defined by user or previously with client
if image is None:
bot.exit("Please load or provide an image.")
# Does the user want to use bind paths option?
if bind is not None:
cmd += self._generate_bind_list(bind)
# Does the user want to run an app?
if app is not None:
cmd = cmd + ["--app", app]
# Does the user want writable?
if writable:
cmd.append("--writable")
# Add options
if options is not None:
cmd = cmd + options
cmd = cmd + [image]
if args is not None:
if not isinstance(args, list):
args = args.split(" ")
cmd = cmd + args
if not stream:
result = self._run_command(cmd, sudo=sudo, return_result=return_result)
else:
return stream_command(cmd, sudo=sudo)
# If the user wants the raw result object
if return_result:
return result
# Otherwise, we parse the result if it was successful
if result:
result = result.strip("\n")
try:
result = json.loads(result)
except:
pass
return result
def build(self,
recipe=None,
image=None,
isolated=False,
sandbox=False,
writable=False,
build_folder=None,
robot_name=False,
ext='sif',
sudo=True,
stream=False,
force=False,
options=None,
quiet=False,
return_result=False):
'''build a singularity image, optionally for an isolated build
(requires sudo). If you specify to stream, expect the image name
and an iterator to be returned.
image, builder = Client.build(...)
Parameters
==========
recipe: the path to the recipe file (or source to build from). If not
defined, we look for "Singularity" file in $PWD
image: the image to build (if None, will use arbitary name
isolated: if True, run build with --isolated flag
sandbox: if True, create a writable sandbox
writable: if True, use writable ext3 (sandbox takes preference)
build_folder: where the container should be built.
ext: the image extension to use.
robot_name: boolean, default False. if you don't give your image a
name (with "image") then a fun robot name will be generated
instead. Highly recommended :)
sudo: give sudo to the command (or not) default is True for build
options: for all other options, specify them in this list.
quiet: quiet verbose printing from the client.
return_result: if True, return complete error code / message dictionary
'''
from spython.utils import check_install
check_install()
cmd = self._init_command('build')
# If no extra options
if not options:
options = []
if 'version 3' in self.version():
ext = 'sif'
# Force the build if the image / sandbox exists
if force:
cmd.append('--force')
# No image provided, default to use the client's loaded image
if recipe is None:
recipe = self._get_uri()
# If it's still None, try default build recipe
if recipe is None:
recipe = 'Singularity'
if not os.path.exists(recipe):
bot.exit('Cannot find %s, exiting.' % image)
if image is None:
if re.search('(docker|shub)://', recipe) and not robot_name:
image = self._get_filename(recipe, ext)
else:
image = "%s.%s" % (self.RobotNamer.generate(), ext)
# Does the user want a custom build folder?
if build_folder is not None:
if not os.path.exists(build_folder):
bot.exit('%s does not exist!' % build_folder)
image = os.path.join(build_folder, image)
# The user wants to run an isolated build
if isolated:
cmd.append('--isolated')
if sandbox:
cmd.append('--sandbox')
elif writable:
cmd.append('--writable')
cmd = cmd + options + [image, recipe]
if not stream:
self._run_command(cmd,
sudo=sudo,
quiet=quiet,
return_result=return_result,
capture=False)
else:
# Here we return the expected image, and an iterator!
# The caller must iterate over
return image, stream_command(cmd, sudo=sudo)
if os.path.exists(image):
return image
def build(self,
recipe=None,
image=None,
isolated=False,
sandbox=False,
writable=False,
build_folder=None,
robot_name=False,
ext='simg',
sudo=True,
stream=False):
'''build a singularity image, optionally for an isolated build
(requires sudo). If you specify to stream, expect the image name
and an iterator to be returned.
image, builder = Client.build(...)
Parameters
==========
recipe: the path to the recipe file (or source to build from). If not
defined, we look for "Singularity" file in $PWD
image: the image to build (if None, will use arbitary name
isolated: if True, run build with --isolated flag
sandbox: if True, create a writable sandbox
writable: if True, use writable ext3 (sandbox takes preference)
build_folder: where the container should be built.
ext: the image extension to use.
robot_name: boolean, default False. if you don't give your image a
name (with "image") then a fun robot name will be generated
instead. Highly recommended :)
sudo: give sudo to the command (or not) default is True for build
'''
self.check_install()
cmd = self._init_command('build')
# No image provided, default to use the client's loaded image
if recipe is None:
recipe = self._get_uri()
# If it's still None, try default build recipe
if recipe is None:
recipe = 'Singularity'
if not os.path.exists(recipe):
bot.exit('Cannot find %s, exiting.' % image)
if image is None:
if re.search('(docker|shub)://', recipe) and robot_name is False:
image = self._get_filename(recipe, ext)
else:
image = "%s.%s" % (self.RobotNamer.generate(), ext)
# Does the user want a custom build folder?
if build_folder is not None:
if not os.path.exists(build_folder):
bot.exit('%s does not exist!' % build_folder)
image = "%s/%s" % (build_folder, image)
if isolated is True:
cmd.append('--isolated')
if sandbox is True:
cmd.append('--sandbox')
elif sandbox is True:
cmd.append('--writable')
cmd = cmd + [image, recipe]
if stream is False:
output = self._run_command(cmd, sudo=sudo, capture=False)
else:
# Here we return the expected image, and an iterator!
# The caller must iterate over
return image, stream_command(cmd, sudo=sudo)
if os.path.exists(image):
return image
def pull(self,
image=None,
name=None,
pull_folder='',
ext="simg",
force=False,
capture=False,
stream=False):
'''pull will pull a singularity hub or Docker image
Parameters
==========
image: the complete image uri. If not provided, the client loaded is used
pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to
user specified location instead.
Docker and Singularity Hub Naming
---------------------------------
name: a custom name to use, to override default
ext: if no name specified, the default extension to use.
'''
from spython.utils import check_install
check_install()
cmd = self._init_command('pull')
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# If it's still None, no go!
if image is None:
bot.exit('You must provide an image uri, or use client.load() first.')
# Singularity Only supports shub and Docker pull
if not re.search('^(shub|docker)://', image):
bot.exit("pull only valid for docker and shub. Use sregistry client.")
# Did the user ask for a custom pull folder?
if pull_folder:
self.setenv('SINGULARITY_PULLFOLDER', pull_folder)
# If we still don't have a custom name, base off of image uri.
if name is None:
name = self._get_filename(image, ext)
cmd = cmd + ["--name", name]
if force is True:
cmd = cmd + ["--force"]
cmd.append(image)
bot.info(' '.join(cmd))
final_image = os.path.join(pull_folder, name)
if stream is False:
self._run_command(cmd, capture=capture)
else:
return final_image, stream_command(cmd, sudo=False)
if os.path.exists(final_image):
bot.info(final_image)
return final_image
def pull(self,
image=None,
name=None,
pull_folder='',
ext=None,
force=False,
capture=False,
stream=False,
quiet=False):
'''pull will pull a singularity hub or Docker image
Parameters
==========
image: the complete image uri. If not provided, the client loaded is used
pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to
user specified location instead.
Docker and Singularity Hub Naming
---------------------------------
name: a custom name to use, to override default
ext: if no name specified, the default extension to use.
'''
from spython.utils import check_install
check_install()
cmd = self._init_command('pull')
# Quiet is honored if set by the client, or user
quiet = quiet or self.quiet
if not ext:
ext = 'sif' if 'version 3' in self.version() else 'simg'
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# If it's still None, no go!
if image is None:
bot.exit('You must provide an image uri, or use client.load() first.')
# Singularity Only supports shub and Docker pull
if not re.search('^(shub|docker)://', image):
bot.exit("pull only valid for docker and shub. Use sregistry client.")
# If we still don't have a custom name, base off of image uri.
if name is None:
name = self._get_filename(image, ext)
if pull_folder:
final_image = os.path.join(pull_folder, os.path.basename(name))
# Regression Singularity 3.* onward, PULLFOLDER not honored
# https://github.com/sylabs/singularity/issues/2788
if 'version 3' in self.version():
name = final_image
pull_folder = None # Don't use pull_folder
else:
final_image = name
cmd = cmd + ["--name", name]
if force:
cmd = cmd + ["--force"]
cmd.append(image)
if not quiet:
bot.info(' '.join(cmd))
with ScopedEnvVar('SINGULARITY_PULLFOLDER', pull_folder):
# Option 1: Streaming we just run to show user
if not stream:
self._run_command(cmd, capture=capture, quiet=quiet)
# Option 3: A custom name we can predict (not commit/hash) and can also show
else:
return final_image, stream_command(cmd, sudo=False)
if os.path.exists(final_image) and not quiet:
bot.info(final_image)
return final_image
def execute(self,
image = None,
command = None,
app = None,
writable = False,
contain = False,
bind = None,
stream = False):
''' execute: send a command to a container
Parameters
==========
image: full path to singularity image
command: command to send to container
app: if not None, execute a command in context of an app
writable: This option makes the file system accessible as read/write
contain: This option disables the automatic sharing of writable
filesystems on your host
bind: list or single string of bind paths.
This option allows you to map directories on your host system to
directories within your container using bind mounts
'''
cmd = self._init_command('exec')
self.check_install()
# If the image is given as a list, it's probably the command
if isinstance(image, list):
command = image
image = None
if command is not None:
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# Does the user want to use bind paths option?
if bind is not None:
cmd += self._generate_bind_list(bind)
# Does the user want to run an app?
if app is not None:
cmd = cmd + ['--app', app]
sudo = False
if writable is True:
sudo = True
if not isinstance(command, list):
command = command.split(' ')
cmd = cmd + [image] + command
if stream is False:
return self._run_command(cmd,sudo=sudo)
return stream_command(cmd, sudo=sudo)
bot.error('Please include a command (list) to execute.')
def execute(
self,
image=None,
command=None,
app=None,
writable=False,
contain=False,
bind=None,
stream=False,
nv=False,
return_result=False,
options=None,
singularity_options=None,
sudo=False,
sudo_options=None,
quiet=True,
environ=None,
):
"""execute: send a command to a container
Parameters
==========
image: full path to singularity image
command: command to send to container
app: if not None, execute a command in context of an app
writable: This option makes the file system accessible as read/write
contain: This option disables the automatic sharing of writable
filesystems on your host
options: an optional list of options to provide to execute.
singularity_options: a list of options to provide to the singularity client
bind: list or single string of bind paths.
This option allows you to map directories on your host system to
directories within your container using bind mounts
nv: if True, load Nvidia Drivers in runtime (default False)
return_result: if True, return entire json object with return code
and message result not (default)
quiet: Do not print verbose output.
environ: extra environment to add.
"""
from spython.utils import check_install
check_install()
cmd = self._init_command("exec", singularity_options)
# nv option leverages any GPU cards
if nv:
cmd += ["--nv"]
# If the image is given as a list, it's probably the command
if isinstance(image, list):
command = image
image = None
if command is not None:
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
self.quiet = True
# If an instance is provided, grab it's name
if isinstance(image, self.instance):
image = image.get_uri()
# If image is still None, not defined by user or previously with client
if image is None:
bot.exit("Please load or provide an image.")
# Does the user want to use bind paths option?
if bind is not None:
cmd += self._generate_bind_list(bind)
# Does the user want to run an app?
if app is not None:
cmd = cmd + ["--app", app]
if writable:
cmd.append("--writable")
# Add additional options
if options is not None:
cmd = cmd + options
if not isinstance(command, list):
command = command.split(" ")
cmd = cmd + [image] + command
# Does the user want to see the command printed?
if not (quiet or self.quiet):
bot.info(" ".join(cmd))
if not stream:
return self._run_command(
cmd,
sudo=sudo,
sudo_options=sudo_options,
return_result=return_result,
quiet=quiet,
environ=environ,
)
return stream_command(cmd, sudo=sudo, sudo_options=sudo_options)
bot.exit("Please include a command (list) to execute.")
def build(
self,
recipe=None,
image=None,
isolated=False,
sandbox=False,
writable=False,
build_folder=None,
robot_name=False,
ext="sif",
sudo=True,
stream=False,
force=False,
options=None,
quiet=False,
return_result=False,
sudo_options=None,
singularity_options=None,
):
"""build a singularity image, optionally for an isolated build
(requires sudo). If you specify to stream, expect the image name
and an iterator to be returned.
image, builder = Client.build(...)
Parameters
==========
recipe: the path to the recipe file (or source to build from). If not
defined, we look for "Singularity" file in $PWD
image: the image to build (if None, will use arbitary name
isolated: if True, run build with --isolated flag
sandbox: if True, create a writable sandbox
writable: if True, use writable ext3 (sandbox takes preference)
build_folder: where the container should be built.
ext: the image extension to use.
robot_name: boolean, default False. if you don't give your image a
name (with "image") then a fun robot name will be generated
instead. Highly recommended :)
sudo: give sudo to the command (or not) default is True for build
sudo_options: options to pass to sudo (e.g. --preserve-env=SINGULARITY_CACHEDIR,SINGULARITY_TMPDIR)
options: for all other options, specify them in this list.
singularity_options: a list of options to provide to the singularity client
quiet: quiet verbose printing from the client.
return_result: if True, return complete error code / message dictionary
"""
from spython.utils import check_install
check_install()
cmd = self._init_command("build", singularity_options)
# If no extra options
options = options or []
if "version 3" in self.version():
ext = "sif"
# Force the build if the image / sandbox exists
if force:
cmd.append("--force")
# No image provided, default to use the client's loaded image
if recipe is None:
recipe = self._get_uri()
# If it's still None, try default build recipe
if recipe is None:
recipe = "Singularity"
if not os.path.exists(recipe):
bot.exit("Cannot find %s, exiting." % image)
if image is None:
if re.search("(docker|shub|library)://", recipe) and not robot_name:
image = self._get_filename(recipe, ext)
else:
image = "%s.%s" % (self.RobotNamer.generate(), ext)
# Does the user want a custom build folder?
if build_folder is not None:
if not os.path.exists(build_folder):
bot.exit("%s does not exist!" % build_folder)
image = os.path.join(build_folder, image)
# The user wants to run an isolated build
if isolated:
cmd.append("--isolated")
if sandbox:
cmd.append("--sandbox")
elif writable:
cmd.append("--writable")
cmd = cmd + options + [image, recipe]
# Does the user want to see the command printed?
if not (quiet or self.quiet):
bot.info(" ".join(cmd))
if not stream:
self._run_command(
cmd,
sudo=sudo,
sudo_options=sudo_options,
quiet=quiet,
return_result=return_result,
capture=False,
)
else:
# Here we return the expected image, and an iterator!
# The caller must iterate over
return image, stream_command(cmd, sudo=sudo, sudo_options=sudo_options)
if os.path.exists(image):
return image
