def validate(self):
"""validate that all (required) fields are included for the Docker
recipe. We minimimally just need a FROM image, and must ensure
it's in a valid format. If anything is missing, we exit with error.
"""
if self.recipe is None:
bot.exit("Please provide a Recipe() to the writer first.")
def convert(self, runscript="/bin/bash", force=False):
"""docker2singularity will return a Singularity build recipe based on
a the loaded recipe object. It doesn't take any arguments as the
recipe object contains the sections, and the calling function
determines saving / output logic.
"""
self.validate()
# Write single recipe that includes all layer
recipe = []
# Number of layers
num_layers = len(self.recipe)
count = 0
# Write each layer to new file
for stage, parser in self.recipe.items():
# Set the first and active stage
self.stage = stage
# From header is required
if parser.fromHeader is None:
bot.exit("Singularity recipe requires a from header.")
recipe += ["\n\n\nBootstrap: docker"]
recipe += ["From: %s" % parser.fromHeader]
recipe += ["Stage: %s\n\n\n" % stage]
# TODO: stopped here - bug with files being found
# Add global files, and then layer files
recipe += self._create_section("files")
for layer, files in parser.layer_files.items():
recipe += create_keyval_section(files, "files", layer)
# Sections with key value pairs
recipe += self._create_section("labels")
recipe += self._create_section("install", "post")
recipe += self._create_section("environ", "environment")
# If we are at the last layer, write the runscript
if count == num_layers - 1:
runscript = self._create_runscript(runscript, force)
# If a working directory was used, add it as a cd
if parser.workdir is not None:
runscript = ["cd " + parser.workdir] + [runscript]
# Finish the recipe, also add as startscript
recipe += finish_section(runscript, "runscript")
recipe += finish_section(runscript, "startscript")
if parser.test is not None:
recipe += finish_section(parser.test, "test")
count += 1
# Clean up extra white spaces
recipe = "\n".join(recipe).replace("\n\n", "\n").strip("\n")
return recipe.rstrip()
def compress(self, image_path):
'''compress will (properly) compress an image'''
if os.path.exists(image_path):
compressed_image = "%s.gz" % image_path
os.system('gzip -c -6 %s > %s' % (image_path, compressed_image))
return compressed_image
bot.exit("Cannot find image %s" % image_path)
def validate(self):
'''validate that all (required) fields are included for the Docker
recipe. We minimimally just need a FROM image, and must ensure
it's in a valid format. If anything is missing, we exit with error.
'''
if self.recipe is None:
bot.exit('Please provide a Recipe() to the writer first.')
if self.recipe.fromHeader is None:
bot.exit("Singularity recipe requires a from header.")
def decompress(self, image_path, quiet=True):
"""decompress will (properly) decompress an image"""
if not os.path.exists(image_path):
bot.exit("Cannot find image %s" % image_path)
extracted_file = image_path.replace(".gz", "")
cmd = ["gzip", "-d", "-f", image_path]
self.run_command(cmd, quiet=quiet) # exits if return code != 0
return extracted_file
def decompress(self, image_path, quiet=True):
'''decompress will (properly) decompress an image'''
if not os.path.exists(image_path):
bot.exit("Cannot find image %s" % image_path)
extracted_file = image_path.replace('.gz', '')
cmd = ['gzip', '-d', '-f', image_path]
result = self.run_command(cmd, quiet=quiet) # exits if return code != 0
return extracted_file
def inspect(self, image=None, json=True, app=None, quiet=True):
'''inspect will show labels, defile, runscript, and tests for an image
Parameters
==========
image: path of image to inspect
json: print json instead of raw text (default True)
quiet: Don't print result to the screen (default True)
app: if defined, return help in context of an app
'''
check_install()
# No image provided, default to use the client's loaded image
if not image:
image = self._get_uri()
# If there still isn't an image, exit on error
if not image:
bot.exit('Please provide an image to inspect.')
cmd = self._init_command('inspect')
if app:
cmd = cmd + ['--app', app]
options = ['e', 'd', 'l', 'r', 'hf', 't']
# After Singularity 3.0, helpfile was changed to H from
if "version 3" in self.version():
options = ['e', 'd', 'l', 'r', 'H', 't']
for x in options:
cmd.append('-%s' % x)
if json:
cmd.append('--json')
cmd.append(image)
result = run_command(cmd, quiet=quiet)
if result['return_code'] == 0:
result = jsonp.loads(result['message'][0])
# Unify output to singularity 3 format
if "data" in result:
result = result['data']
# Fix up labels
result = parse_labels(result)
if not quiet:
print(jsonp.dumps(result, indent=4))
return result
def _run_checks(self):
"""basic sanity checks for the file name (and others if needed) before
attempting parsing.
"""
if self.filename is not None:
# Does the recipe provided exist?
if not os.path.exists(self.filename):
bot.exit("Cannot find %s, is the path correct?" % self.filename)
# Ensure we carry fullpath
self.filename = os.path.abspath(self.filename)
def validate_stage(self, parser):
"""Given a recipe parser for a stage, ensure that the recipe is valid"""
if parser.fromHeader is None:
bot.exit("Dockerfile requires a fromHeader.")
# Parse the provided name
uri_regexes = [_reduced_uri, _default_uri, _docker_uri]
for r in uri_regexes:
match = r.match(parser.fromHeader)
if match:
break
if not match:
bot.exit("FROM header %s not valid." % parser.fromHeader)
def pull(self, image=None, name=None, pull_folder='', ext="simg"):
'''pull will pull a singularity hub or Docker image
Parameters
==========
image: the complete image uri. If not provided, the client loaded is used
pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to
user specified location instead.
Docker and Singularity Hub Naming
---------------------------------
name: a custom name to use, to override default
ext: if no name specified, the default extension to use.
'''
self.check_install()
cmd = self._init_command('pull')
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# If it's still None, no go!
if image is None:
bot.exit('You must provide an image uri, or use client.load() first.')
# Singularity Only supports shub and Docker pull
if not re.search('^(shub|docker)://', image):
bot.exit("pull only valid for docker and shub. Use sregistry client.")
# Did the user ask for a custom pull folder?
if pull_folder:
self.setenv('SINGULARITY_PULLFOLDER', pull_folder)
# If we still don't have a custom name, base off of image uri.
if name is None:
name = self._get_filename(image, ext)
cmd = cmd + ["--name", name]
cmd.append(image)
bot.info(' '.join(cmd))
self._run_command(cmd, capture=False)
final_image = os.path.join(pull_folder, name)
if os.path.exists(final_image):
bot.info(final_image)
return final_image
def get_container_id(self, container_id=None):
"""a helper function shared between functions that will return a
container_id. First preference goes to a container_id provided by
the user at runtime. Second preference goes to the container_id
instantiated with the client.
Parameters
==========
container_id: image uri to parse (required)
"""
# The user must provide a container_id, or have one with the client
if container_id is None and self.container_id is None:
bot.exit("You must provide a container_id.")
# Choose whichever is not None, with preference for function provided
container_id = container_id or self.container_id
return container_id
def create(self, image_path, size=1024, sudo=False):
'''create will create a a new image
Parameters
==========
image_path: full path to image
size: image sizein MiB, default is 1024MiB
filesystem: supported file systems ext3/ext4 (ext[2/3]: default ext3
'''
self.check_install()
cmd = self.init_command('image.create')
cmd = cmd + ['--size', str(size), image_path]
output = self.run_command(cmd, sudo=sudo)
self.println(output)
if not os.path.exists(image_path):
bot.exit("Could not create image %s" % image_path)
return image_path
def export(self, image_path, tmptar=None):
'''export will export an image, sudo must be used.
Parameters
==========
image_path: full path to image
tmptar: if defined, use custom temporary path for tar export
'''
from spython.utils import check_install
check_install()
if 'version 3' in self.version():
bot.exit('export is deprecated after Singularity 2.*')
if tmptar is None:
tmptar = "/%s/tmptar.tar" %(tempfile.mkdtemp())
cmd = ['singularity', 'image.export', '-f', tmptar, image_path]
self.run_command(cmd, sudo=False)
return tmptar
def write(self, output_file=None, force=False):
'''convert a recipe to a specified format, and write to file, meaning
we use the loaded recipe to write to an output file.
If the output file is not specified, a temporary file is used.
Parameters
==========
output_file: the file to save to, not required (estimates default)
force: if True, if file exists, over-write existing file
'''
if output_file is None:
output_file = self._get_conversion_outfile()
# Cut out early if file exists and we aren't overwriting
if os.path.exists(output_file) and not force:
bot.exit('%s exists, and force is False.' % output_file)
# Do the conversion if function is provided by subclass
if hasattr(self, 'convert'):
converted = self.convert()
bot.info('Saving to %s' % output_file)
write_file(output_file, converted)
def create(self, image_path, size=1024, sudo=False, singularity_options=None):
"""create will create a a new image
Parameters
==========
image_path: full path to image
size: image sizein MiB, default is 1024MiB
filesystem: supported file systems ext3/ext4 (ext[2/3]: default ext3
singularity_options: a list of options to provide to the singularity client
"""
from spython.utils import check_install
check_install()
cmd = self.init_command("image.create", singularity_options)
cmd = cmd + ["--size", str(size), image_path]
output = self.run_command(cmd, sudo=sudo)
self.println(output)
if not os.path.exists(image_path):
bot.exit("Could not create image %s" % image_path)
return image_path
def validate(self):
"""validate that all (required) fields are included for the Docker
recipe. We minimimally just need a FROM image, and must ensure
it's in a valid format. If anything is missing, we exit with error.
"""
if self.recipe is None:
bot.exit("Please provide a Recipe() to the writer first.")
if self.recipe.fromHeader is None:
bot.exit("Dockerfile requires a fromHeader.")
# Parse the provided name
uri_regexes = [_reduced_uri, _default_uri, _docker_uri]
for r in uri_regexes:
match = r.match(self.recipe.fromHeader)
if match:
break
if not match:
bot.exit("FROM header %s not valid." % self.recipe.fromHeader)
def start(self,
image=None,
name=None,
args=None,
sudo=False,
options=None,
capture=False):
"""start an instance. This is done by default when an instance is created.
Parameters
==========
image: optionally, an image uri (if called as a command from Client)
name: a name for the instance
sudo: if the user wants to run the command with sudo
capture: capture output, default is False. With True likely to hang.
args: arguments to provide to the instance (supported Singularity 3.1+)
options: a list of tuples, each an option to give to the start command
[("--bind", "/tmp"),...]
USAGE:
singularity [...] instance.start [...] <container path> <instance name>
"""
from spython.utils import run_command, check_install
check_install()
# If name provided, over write robot (default)
if name is not None:
self.name = name
# If an image isn't provided, we have an initialized instance
if image is None:
# Not having this means it was called as a command, without an image
if not hasattr(self, "_image"):
bot.exit("Please provide an image, or create an Instance first.")
image = self._image
# Derive subgroup command based on singularity version
subgroup = "instance.start"
if "version 3" in self.version():
subgroup = ["instance", "start"]
cmd = self._init_command(subgroup)
# Add options, if they are provided
if not isinstance(options, list):
options = [] if options is None else options.split(" ")
# Assemble the command!
cmd = cmd + options + [image, self.name]
# If arguments are provided
if args is not None:
if not isinstance(args, list):
args = [args]
cmd = cmd + args
# Save the options and cmd, if the user wants to see them later
self.options = options
self.args = args
self.cmd = cmd
output = run_command(cmd, sudo=sudo, quiet=True, capture=capture)
if output["return_code"] == 0:
self._update_metadata()
else:
message = "%s : return code %s" % (output["message"],
output["return_code"])
bot.error(message)
return self
def execute(
self,
image=None,
command=None,
app=None,
writable=False,
contain=False,
bind=None,
stream=False,
nv=False,
return_result=False,
options=None,
singularity_options=None,
sudo=False,
sudo_options=None,
quiet=True,
environ=None,
):
"""execute: send a command to a container
Parameters
==========
image: full path to singularity image
command: command to send to container
app: if not None, execute a command in context of an app
writable: This option makes the file system accessible as read/write
contain: This option disables the automatic sharing of writable
filesystems on your host
options: an optional list of options to provide to execute.
singularity_options: a list of options to provide to the singularity client
bind: list or single string of bind paths.
This option allows you to map directories on your host system to
directories within your container using bind mounts
nv: if True, load Nvidia Drivers in runtime (default False)
return_result: if True, return entire json object with return code
and message result not (default)
quiet: Do not print verbose output.
environ: extra environment to add.
"""
from spython.utils import check_install
check_install()
cmd = self._init_command("exec", singularity_options)
# nv option leverages any GPU cards
if nv:
cmd += ["--nv"]
# If the image is given as a list, it's probably the command
if isinstance(image, list):
command = image
image = None
if command is not None:
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
self.quiet = True
# If an instance is provided, grab it's name
if isinstance(image, self.instance):
image = image.get_uri()
# If image is still None, not defined by user or previously with client
if image is None:
bot.exit("Please load or provide an image.")
# Does the user want to use bind paths option?
if bind is not None:
cmd += self._generate_bind_list(bind)
# Does the user want to run an app?
if app is not None:
cmd = cmd + ["--app", app]
if writable:
cmd.append("--writable")
# Add additional options
if options is not None:
cmd = cmd + options
if not isinstance(command, list):
command = command.split(" ")
cmd = cmd + [image] + command
# Does the user want to see the command printed?
if not (quiet or self.quiet):
bot.info(" ".join(cmd))
if not stream:
return self._run_command(
cmd,
sudo=sudo,
sudo_options=sudo_options,
return_result=return_result,
quiet=quiet,
environ=environ,
)
return stream_command(cmd, sudo=sudo, sudo_options=sudo_options)
bot.exit("Please include a command (list) to execute.")
def build(self,
recipe=None,
image=None,
isolated=False,
sandbox=False,
writable=False,
build_folder=None,
robot_name=False,
ext='sif',
sudo=True,
stream=False,
force=False,
options=None,
quiet=False,
return_result=False):
'''build a singularity image, optionally for an isolated build
(requires sudo). If you specify to stream, expect the image name
and an iterator to be returned.
image, builder = Client.build(...)
Parameters
==========
recipe: the path to the recipe file (or source to build from). If not
defined, we look for "Singularity" file in $PWD
image: the image to build (if None, will use arbitary name
isolated: if True, run build with --isolated flag
sandbox: if True, create a writable sandbox
writable: if True, use writable ext3 (sandbox takes preference)
build_folder: where the container should be built.
ext: the image extension to use.
robot_name: boolean, default False. if you don't give your image a
name (with "image") then a fun robot name will be generated
instead. Highly recommended :)
sudo: give sudo to the command (or not) default is True for build
options: for all other options, specify them in this list.
quiet: quiet verbose printing from the client.
return_result: if True, return complete error code / message dictionary
'''
from spython.utils import check_install
check_install()
cmd = self._init_command('build')
# If no extra options
if not options:
options = []
if 'version 3' in self.version():
ext = 'sif'
# Force the build if the image / sandbox exists
if force:
cmd.append('--force')
# No image provided, default to use the client's loaded image
if recipe is None:
recipe = self._get_uri()
# If it's still None, try default build recipe
if recipe is None:
recipe = 'Singularity'
if not os.path.exists(recipe):
bot.exit('Cannot find %s, exiting.' % image)
if image is None:
if re.search('(docker|shub)://', recipe) and not robot_name:
image = self._get_filename(recipe, ext)
else:
image = "%s.%s" % (self.RobotNamer.generate(), ext)
# Does the user want a custom build folder?
if build_folder is not None:
if not os.path.exists(build_folder):
bot.exit('%s does not exist!' % build_folder)
image = os.path.join(build_folder, image)
# The user wants to run an isolated build
if isolated:
cmd.append('--isolated')
if sandbox:
cmd.append('--sandbox')
elif writable:
cmd.append('--writable')
cmd = cmd + options + [image, recipe]
if not stream:
self._run_command(cmd,
sudo=sudo,
quiet=quiet,
return_result=return_result,
capture=False)
else:
# Here we return the expected image, and an iterator!
# The caller must iterate over
return image, stream_command(cmd, sudo=sudo)
if os.path.exists(image):
return image
def build(self,
recipe=None,
image=None,
isolated=False,
sandbox=False,
writable=False,
build_folder=None,
robot_name=False,
ext='simg',
sudo=True,
stream=False):
'''build a singularity image, optionally for an isolated build
(requires sudo). If you specify to stream, expect the image name
and an iterator to be returned.
image, builder = Client.build(...)
Parameters
==========
recipe: the path to the recipe file (or source to build from). If not
defined, we look for "Singularity" file in $PWD
image: the image to build (if None, will use arbitary name
isolated: if True, run build with --isolated flag
sandbox: if True, create a writable sandbox
writable: if True, use writable ext3 (sandbox takes preference)
build_folder: where the container should be built.
ext: the image extension to use.
robot_name: boolean, default False. if you don't give your image a
name (with "image") then a fun robot name will be generated
instead. Highly recommended :)
sudo: give sudo to the command (or not) default is True for build
'''
self.check_install()
cmd = self._init_command('build')
# No image provided, default to use the client's loaded image
if recipe is None:
recipe = self._get_uri()
# If it's still None, try default build recipe
if recipe is None:
recipe = 'Singularity'
if not os.path.exists(recipe):
bot.exit('Cannot find %s, exiting.' % image)
if image is None:
if re.search('(docker|shub)://', recipe) and robot_name is False:
image = self._get_filename(recipe, ext)
else:
image = "%s.%s" % (self.RobotNamer.generate(), ext)
# Does the user want a custom build folder?
if build_folder is not None:
if not os.path.exists(build_folder):
bot.exit('%s does not exist!' % build_folder)
image = "%s/%s" % (build_folder, image)
if isolated is True:
cmd.append('--isolated')
if sandbox is True:
cmd.append('--sandbox')
elif sandbox is True:
cmd.append('--writable')
cmd = cmd + [image, recipe]
if stream is False:
output = self._run_command(cmd, sudo=sudo, capture=False)
else:
# Here we return the expected image, and an iterator!
# The caller must iterate over
return image, stream_command(cmd, sudo=sudo)
if os.path.exists(image):
return image
def inspect(self,
image=None,
json=True,
app=None,
quiet=True,
singularity_options=None):
"""inspect will show labels, defile, runscript, and tests for an image
Parameters
==========
image: path of image to inspect
json: print json instead of raw text (default True)
quiet: Don't print result to the screen (default True)
app: if defined, return help in context of an app
singularity_options: a list of options to provide to the singularity client
"""
check_install()
# No image provided, default to use the client's loaded image
if not image:
image = self._get_uri()
# If there still isn't an image, exit on error
if not image:
bot.exit("Please provide an image to inspect.")
cmd = self._init_command("inspect", singularity_options)
if app:
cmd = cmd + ["--app", app]
options = ["e", "d", "l", "r", "hf", "t"]
# After Singularity 3.0, helpfile was changed to H from
if "version 3" in self.version():
options = ["e", "d", "l", "r", "H", "t"]
for x in options:
cmd.append("-%s" % x)
if json:
cmd.append("--json")
cmd.append(image)
result = run_command(cmd, quiet=quiet)
if result["return_code"] == 0:
result = jsonp.loads(result["message"][0])
# Unify output to singularity 3 format
if "data" in result:
result = result["data"]
# Fix up labels
result = parse_labels(result)
if not quiet:
print(jsonp.dumps(result, indent=4))
return result
def list_instances(
self,
name=None,
return_json=False,
quiet=False,
sudo=False,
sudo_options=None,
singularity_options=None,
):
"""list instances. For Singularity, this is provided as a command sub
group.
singularity instance list
Return codes provided are different from standard linux:
see https://github.com/singularityware/singularity/issues/1706
Since we expect json output, we don't support older versions of Singularity.
Parameters
==========
return_json: return a json list of instances instead of objects (False)
name: if defined, return the list for just one instance (used to ged pid)
singularity_options: a list of options to provide to the singularity client
Return Code -- Reason
0 -- Instances Found
1 -- No Instances, libexecdir value not found, functions file not found
255 -- Couldn't get UID
"""
from spython.utils import check_install
check_install()
subgroup = ["instance", "list", "--json"]
if "version 3" not in self.version():
bot.exit("This version of Singularity Python does not support < 3.0.")
cmd = self._init_command(subgroup, singularity_options)
# If the user has provided a name, we want to see a particular instance
if name is not None:
cmd.append(name)
# Does the user want to see the command printed?
if not (quiet or self.quiet):
bot.info(" ".join(cmd))
output = run_command(cmd, quiet=True, sudo=sudo, sudo_options=sudo_options)
instances = []
# Success, we have instances
if output["return_code"] == 0:
instances = json.loads(output["message"][0]).get("instances", {})
# Does the user want instance objects instead?
listing = []
if not return_json:
for i in instances:
# If the user has provided a name, only add instance matches
if name is not None:
if name != i["instance"]:
continue
# Otherwise, add instances to the listing
new_instance = self.instance(
pid=i.get("pid"),
ip_address=i.get("ip"),
name=i.get("instance") or i.get("daemon_name"),
log_err_path=i.get("logErrPath"),
log_out_path=i.get("logOutPath"),
image=i.get("img") or i.get("container_image"),
start=False,
)
listing.append(new_instance)
instances = listing
# Couldn't get UID
elif output["return_code"] == 255:
bot.error("Couldn't get UID")
# Return code of 0
else:
bot.info("No instances found.")
# If we are given a name, return just one
if name is not None and instances and len(instances) == 1:
instances = instances[0]
return instances
def _check_install(self):
"""ensure that singularity is installed, and exit if not.
"""
if check_install() is not True:
bot.exit("Cannot find Singularity! Is it installed?")
def _run(
self,
bundle,
container_id=None,
empty_process=False,
log_path=None,
pid_file=None,
sync_socket=None,
command="run",
log_format="kubernetes",
singularity_options=None,
):
"""_run is the base function for run and create, the only difference
between the two being that run does not have an option for sync_socket.
Equivalent command line example:
singularity oci create [create options...] <container_ID>
Parameters
==========
bundle: the full path to the bundle folder
container_id: an optional container_id. If not provided, use same
container_id used to generate OciImage instance
empty_process: run container without executing container process (for
example, for a pod container waiting for signals). This
is a specific use case for tools like Kubernetes
log_path: the path to store the log.
pid_file: specify the pid file path to use
sync_socket: the path to the unix socket for state synchronization.
command: the command (run or create) to use (default is run)
log_format: defaults to kubernetes. Can also be "basic" or "json"
singularity_options: a list of options to provide to the singularity client
"""
container_id = self.get_container_id(container_id)
# singularity oci create
cmd = self._init_command(command, singularity_options)
# Check that the bundle exists
if not os.path.exists(bundle):
bot.exit("Bundle not found at %s" % bundle)
# Add the bundle
cmd = cmd + ["--bundle", bundle]
# Additional Logging Files
cmd = cmd + ["--log-format", log_format]
if log_path is not None:
cmd = cmd + ["--log-path", log_path]
if pid_file is not None:
cmd = cmd + ["--pid-file", pid_file]
if sync_socket is not None:
cmd = cmd + ["--sync-socket", sync_socket]
if empty_process:
cmd.append("--empty-process")
# Finally, add the container_id
cmd.append(container_id)
# Generate the instance
self._send_command(cmd, sudo=True)
# Get the status to report to the user!
return self.state(container_id, sudo=True, sync_socket=sync_socket)
def main(args, options, parser):
"""This function serves as a wrapper around the DockerParser,
SingularityParser, DockerWriter, and SingularityParser converters.
We can either save to file if args.outfile is defined, or print
to the console if not.
"""
# We need something to work with
if not args.files:
parser.print_help()
sys.exit(1)
# Get the user specified input and output files
outfile = None
if len(args.files) > 1:
outfile = args.files[1]
# First try to get writer and parser, if not defined will return None
writer = get_writer(args.writer)
parser = get_parser(args.parser)
# If the user wants to auto-detect the type
if args.parser == "auto":
if "dockerfile" in args.files[0].lower():
parser = get_parser("docker")
elif "singularity" in args.files[0].lower():
parser = get_parser("singularity")
# If the parser still isn't defined, no go.
if parser is None:
bot.exit(
"Please provide a Dockerfile or Singularity recipe, or define the --parser type."
)
# If the writer needs auto-detect
if args.writer == "auto":
if parser.name == "docker":
writer = get_writer("singularity")
else:
writer = get_writer("docker")
# If the writer still isn't defined, no go
if writer is None:
bot.exit("Please define the --writer type.")
# Initialize the chosen parser
recipeParser = parser(args.files[0])
# By default, discover entrypoint / cmd from Dockerfile
entrypoint = "/bin/bash"
force = False
if args.entrypoint is not None:
entrypoint = args.entrypoint
# This is only done if the user intended to print json here
recipeParser.entrypoint = args.entrypoint
recipeParser.cmd = None
force = True
if args.json:
if outfile is not None:
if not os.path.exists(outfile):
if force:
write_json(outfile, recipeParser.recipe.json())
else:
bot.exit("%s exists, set --force to overwrite." % outfile)
else:
print(json.dumps(recipeParser.recipe.json(), indent=4))
else:
# Do the conversion
recipeWriter = writer(recipeParser.recipe)
result = recipeWriter.convert(runscript=entrypoint, force=force)
# If the user specifies an output file, save to it
if outfile is not None:
write_file(outfile, result)
# Otherwise, convert and print to screen
else:
print(result)
def run(
self,
image=None,
args=None,
app=None,
sudo=False,
writable=False,
contain=False,
bind=None,
stream=False,
nv=False,
options=None,
return_result=False,
):
"""
run will run the container, with or withour arguments (which
should be provided in a list)
Parameters
==========
image: full path to singularity image
args: args to include with the run
app: if not None, execute a command in context of an app
writable: This option makes the file system accessible as read/write
options: an optional list of options to provide to run.
contain: This option disables the automatic sharing of writable
filesystems on your host
bind: list or single string of bind paths.
This option allows you to map directories on your host system to
directories within your container using bind mounts
stream: if True, return <generator> for the user to run
nv: if True, load Nvidia Drivers in runtime (default False)
return_result: if True, return entire json object with return code
and message result (default is False)
"""
from spython.utils import check_install
check_install()
cmd = self._init_command("run")
# nv option leverages any GPU cards
if nv:
cmd += ["--nv"]
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# If an instance is provided, grab it's name
if isinstance(image, self.instance):
image = image.get_uri()
# If image is still None, not defined by user or previously with client
if image is None:
bot.exit("Please load or provide an image.")
# Does the user want to use bind paths option?
if bind is not None:
cmd += self._generate_bind_list(bind)
# Does the user want to run an app?
if app is not None:
cmd = cmd + ["--app", app]
# Does the user want writable?
if writable:
cmd.append("--writable")
# Add options
if options is not None:
cmd = cmd + options
cmd = cmd + [image]
if args is not None:
if not isinstance(args, list):
args = args.split(" ")
cmd = cmd + args
if not stream:
result = self._run_command(cmd, sudo=sudo, return_result=return_result)
else:
return stream_command(cmd, sudo=sudo)
# If the user wants the raw result object
if return_result:
return result
# Otherwise, we parse the result if it was successful
if result:
result = result.strip("\n")
try:
result = json.loads(result)
except:
pass
return result
def pull(self,
image=None,
name=None,
pull_folder='',
ext=None,
force=False,
capture=False,
stream=False,
quiet=False):
'''pull will pull a singularity hub or Docker image
Parameters
==========
image: the complete image uri. If not provided, the client loaded is used
pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to
user specified location instead.
Docker and Singularity Hub Naming
---------------------------------
name: a custom name to use, to override default
ext: if no name specified, the default extension to use.
'''
from spython.utils import check_install
check_install()
cmd = self._init_command('pull')
# Quiet is honored if set by the client, or user
quiet = quiet or self.quiet
if not ext:
ext = 'sif' if 'version 3' in self.version() else 'simg'
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# If it's still None, no go!
if image is None:
bot.exit('You must provide an image uri, or use client.load() first.')
# Singularity Only supports shub and Docker pull
if not re.search('^(shub|docker)://', image):
bot.exit("pull only valid for docker and shub. Use sregistry client.")
# If we still don't have a custom name, base off of image uri.
if name is None:
name = self._get_filename(image, ext)
if pull_folder:
final_image = os.path.join(pull_folder, os.path.basename(name))
# Regression Singularity 3.* onward, PULLFOLDER not honored
# https://github.com/sylabs/singularity/issues/2788
if 'version 3' in self.version():
name = final_image
pull_folder = None # Don't use pull_folder
else:
final_image = name
cmd = cmd + ["--name", name]
if force:
cmd = cmd + ["--force"]
cmd.append(image)
if not quiet:
bot.info(' '.join(cmd))
with ScopedEnvVar('SINGULARITY_PULLFOLDER', pull_folder):
# Option 1: Streaming we just run to show user
if not stream:
self._run_command(cmd, capture=capture, quiet=quiet)
# Option 3: A custom name we can predict (not commit/hash) and can also show
else:
return final_image, stream_command(cmd, sudo=False)
if os.path.exists(final_image) and not quiet:
bot.info(final_image)
return final_image
def pull(self,
image=None,
name=None,
pull_folder='',
ext="simg",
force=False,
capture=False,
name_by_commit=False,
name_by_hash=False,
stream=False):
'''pull will pull a singularity hub or Docker image
Parameters
==========
image: the complete image uri. If not provided, the client loaded is used
pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to
user specified location instead.
Docker and Singularity Hub Naming
---------------------------------
name: a custom name to use, to override default
ext: if no name specified, the default extension to use.
'''
from spython.utils import check_install
check_install()
cmd = self._init_command('pull')
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# If it's still None, no go!
if image is None:
bot.exit('You must provide an image uri, or use client.load() first.')
# Singularity Only supports shub and Docker pull
if not re.search('^(shub|docker)://', image):
bot.exit("pull only valid for docker and shub. Use sregistry client.")
# Did the user ask for a custom pull folder?
if pull_folder:
self.setenv('SINGULARITY_PULLFOLDER', pull_folder)
# If we still don't have a custom name, base off of image uri.
# Determine how to tell client to name the image, preference to hash
if name_by_hash is True:
cmd.append('--hash')
elif name_by_commit is True:
cmd.append('--commit')
elif name is None:
name = self._get_filename(image, ext)
# Only add name if we aren't naming by hash or commit
if not name_by_commit and not name_by_hash:
cmd = cmd + ["--name", name]
if force is True:
cmd = cmd + ["--force"]
cmd.append(image)
bot.info(' '.join(cmd))
# If name is still None, make empty string
if name is None:
name = ''
final_image = os.path.join(pull_folder, name)
# Option 1: For hash or commit, need return value to get final_image
if name_by_commit or name_by_hash:
# Set pull to temporary location
tmp_folder = tempfile.mkdtemp()
self.setenv('SINGULARITY_PULLFOLDER', tmp_folder)
self._run_command(cmd, capture=capture)
try:
tmp_image = os.path.join(tmp_folder, os.listdir(tmp_folder)[0])
final_image = os.path.join(pull_folder,
os.path.basename(tmp_image))
shutil.move(tmp_image, final_image)
shutil.rmtree(tmp_folder)
except:
bot.error('Issue pulling image with commit or hash, try without?')
# Option 2: Streaming we just run to show user
elif stream is False:
self._run_command(cmd, capture=capture)
# Option 3: A custom name we can predict (not commit/hash) and can also show
else:
return final_image, stream_command(cmd, sudo=False)
if os.path.exists(final_image):
bot.info(final_image)
return final_image
def build(
self,
recipe=None,
image=None,
isolated=False,
sandbox=False,
writable=False,
build_folder=None,
robot_name=False,
ext="sif",
sudo=True,
stream=False,
force=False,
options=None,
quiet=False,
return_result=False,
sudo_options=None,
singularity_options=None,
):
"""build a singularity image, optionally for an isolated build
(requires sudo). If you specify to stream, expect the image name
and an iterator to be returned.
image, builder = Client.build(...)
Parameters
==========
recipe: the path to the recipe file (or source to build from). If not
defined, we look for "Singularity" file in $PWD
image: the image to build (if None, will use arbitary name
isolated: if True, run build with --isolated flag
sandbox: if True, create a writable sandbox
writable: if True, use writable ext3 (sandbox takes preference)
build_folder: where the container should be built.
ext: the image extension to use.
robot_name: boolean, default False. if you don't give your image a
name (with "image") then a fun robot name will be generated
instead. Highly recommended :)
sudo: give sudo to the command (or not) default is True for build
sudo_options: options to pass to sudo (e.g. --preserve-env=SINGULARITY_CACHEDIR,SINGULARITY_TMPDIR)
options: for all other options, specify them in this list.
singularity_options: a list of options to provide to the singularity client
quiet: quiet verbose printing from the client.
return_result: if True, return complete error code / message dictionary
"""
from spython.utils import check_install
check_install()
cmd = self._init_command("build", singularity_options)
# If no extra options
options = options or []
if "version 3" in self.version():
ext = "sif"
# Force the build if the image / sandbox exists
if force:
cmd.append("--force")
# No image provided, default to use the client's loaded image
if recipe is None:
recipe = self._get_uri()
# If it's still None, try default build recipe
if recipe is None:
recipe = "Singularity"
if not os.path.exists(recipe):
bot.exit("Cannot find %s, exiting." % image)
if image is None:
if re.search("(docker|shub|library)://", recipe) and not robot_name:
image = self._get_filename(recipe, ext)
else:
image = "%s.%s" % (self.RobotNamer.generate(), ext)
# Does the user want a custom build folder?
if build_folder is not None:
if not os.path.exists(build_folder):
bot.exit("%s does not exist!" % build_folder)
image = os.path.join(build_folder, image)
# The user wants to run an isolated build
if isolated:
cmd.append("--isolated")
if sandbox:
cmd.append("--sandbox")
elif writable:
cmd.append("--writable")
cmd = cmd + options + [image, recipe]
# Does the user want to see the command printed?
if not (quiet or self.quiet):
bot.info(" ".join(cmd))
if not stream:
self._run_command(
cmd,
sudo=sudo,
sudo_options=sudo_options,
quiet=quiet,
return_result=return_result,
capture=False,
)
else:
# Here we return the expected image, and an iterator!
# The caller must iterate over
return image, stream_command(cmd, sudo=sudo, sudo_options=sudo_options)
if os.path.exists(image):
return image
