def get(self, name, return_json=False, quiet=False, singularity_options=None):
"""get is a list for a single instance. It is assumed to be running,
and we need to look up the PID, etc.
"""
from spython.utils import check_install
check_install()
# Ensure compatible for singularity prior to 3.0, and after 3.0
subgroup = "instance.list"
if "version 3" in self.version():
subgroup = ["instance", "list"]
cmd = self._init_command(subgroup, singularity_options)
cmd.append(name)
output = self.run_command(cmd, quiet=True)
# Success, we have instances
if output["return_code"] == 0:
# Only print the table if we are returning json
if not quiet:
print("".join(output["message"]))
# Prepare json result from table
header = ["daemon_name", "pid", "container_image"]
instances = parse_table(output["message"][0], header)
# Does the user want instance objects instead?
listing = []
if not return_json:
for i in instances:
new_instance = Instance(
pid=i["pid"],
name=i["daemon_name"],
image=i["container_image"],
start=False,
)
listing.append(new_instance)
instances = listing
# Couldn't get UID
elif output["return_code"] == 255:
bot.error("Couldn't get UID")
# Return code of 0
else:
bot.info("No instances found.")
# If we are given a name, return just one
if name is not None and len(instances) == 1:
instances = instances[0]
return instances
def run_command(self, cmd, sudo=False, quiet=False, capture=True):
'''run_command is a wrapper for the global run_command, checking first
for sudo and exiting on error if needed. The message is returned as
a list of lines for the calling function to parse, and stdout uses
the parent process so it appears for the user.
Parameters
==========
cmd: the command to run
sudo: does the command require sudo?
On success, returns result. Otherwise, exists on error
'''
result = run_cmd(cmd, sudo=sudo, capture=capture)
message = result['message']
return_code = result['return_code']
if result['return_code'] == 0:
if len(message) == 1:
message = message[0]
return message
if quiet is False:
bot.error("Return Code %s: %s" %(return_code,
message))
def _load_bootstrap(self, line):
'''load bootstrap checks that the bootstrap is Docker, otherwise we
exit on fail (there is no other option to convert to Dockerfile!
'''
if 'docker' not in line.lower():
bot.error('docker not detected as Bootstrap!')
sys.exit(1)
def stopall(self, sudo=False, quiet=True):
"""stop ALL instances. This command is only added to the command group
as it doesn't make sense to call from a single instance
Parameters
==========
sudo: if the command should be done with sudo (exposes different set of
instances)
"""
from spython.utils import check_install
check_install()
subgroup = "instance.stop"
if "version 3" in self.version():
subgroup = ["instance", "stop"]
cmd = self._init_command(subgroup)
cmd = cmd + ["--all"]
output = run_command(cmd, sudo=sudo, quiet=quiet)
if output["return_code"] != 0:
message = "%s : return code %s" % (output["message"],
output["return_code"])
bot.error(message)
return output["return_code"]
return output["return_code"]
def stop(self, name=None, sudo=False):
'''start an instance. This is done by default when an instance is created.
Parameters
==========
image: optionally, an image uri (if called as a command from Client)
name: a name for the instance
sudo: if the user wants to run the command with sudo
USAGE:
singularity [...] instance.start [...] <container path> <instance name>
'''
from spython.utils import (check_install, run_command)
check_install()
cmd = self._init_command('instance.stop')
# If name is provided assume referencing an instance
instance_name = self.name
if name is not None:
instance_name = name
cmd = cmd + [instance_name]
output = run_command(cmd, sudo=sudo, quiet=True)
if output['return_code'] != 0:
message = '%s : return code %s' % (output['message'],
output['return_code'])
bot.error(message)
return output['return_code']
return output['return_code']
def execute(self,
image=None,
command=None,
app=None,
writable=False,
contain=False,
bind=None):
''' execute: send a command to a container
Parameters
==========
image: full path to singularity image
command: command to send to container
app: if not None, execute a command in context of an app
writable: This option makes the file system accessible as read/write
contain: This option disables the automatic sharing of writable
filesystems on your host
bind: list or single string of bind paths.
This option allows you to map directories on your host system to
directories within your container using bind mounts
'''
cmd = self._init_command('exec')
self.check_install()
# If the image is given as a list, it's probably the command
if isinstance(image, list):
command = image
image = None
if command is not None:
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# Does the user want to use bind paths option?
if bind is not None:
cmd += self._generate_bind_list(bind)
# Does the user want to run an app?
if app is not None:
cmd = cmd + ['--app', app]
sudo = False
if writable is True:
sudo = True
if not isinstance(command, list):
command = command.split(' ')
cmd = cmd + [image] + command
return self._run_command(cmd,sudo=sudo)
bot.error('Please include a command (list) to execute.')
def start(self, image=None, name=None, sudo=False, options=[], capture=False):
'''start an instance. This is done by default when an instance is created.
Parameters
==========
image: optionally, an image uri (if called as a command from Client)
name: a name for the instance
sudo: if the user wants to run the command with sudo
capture: capture output, default is False. With True likely to hang.
options: a list of tuples, each an option to give to the start command
[("--bind", "/tmp"),...]
USAGE:
singularity [...] instance.start [...] <container path> <instance name>
'''
from spython.utils import (run_command, check_install)
check_install()
# If no name provided, give it an excellent one!
if name is None:
name = self.RobotNamer.generate()
self.name = name.replace('-', '_')
# If an image isn't provided, we have an initialized instance
if image is None:
# Not having this means it was called as a command, without an image
if not hasattr(self, "_image"):
bot.error('Please provide an image, or create an Instance first.')
sys.exit(1)
image = self._image
cmd = self._init_command('instance.start')
# Add options, if they are provided
if not isinstance(options, list):
options = options.split(' ')
# Assemble the command!
cmd = cmd + options + [image, self.name]
# Save the options and cmd, if the user wants to see them later
self.options = options
self.cmd = cmd
output = run_command(cmd, sudo=sudo, quiet=True, capture=capture)
if output['return_code'] == 0:
self._update_metadata()
else:
message = '%s : return code %s' % (output['message'],
output['return_code'])
bot.error(message)
return self
def stop(
self,
name=None,
sudo=False,
sudo_options=None,
timeout=None,
singularity_options=None,
quiet=True,
):
"""stop an instance. This is done by default when an instance is created.
Parameters
==========
name: a name for the instance
sudo: if the user wants to run the command with sudo
singularity_options: a list of options to provide to the singularity client
quiet: Do not print verbose output.
timeout: forcebly kill non-stopped instance after the
timeout specified in seconds
USAGE:
singularity [...] instance.stop [...] <instance name>
"""
from spython.utils import check_install, run_command
check_install()
subgroup = "instance.stop"
if "version 3" in self.version():
subgroup = ["instance", "stop"]
if timeout:
subgroup += ["-t", str(timeout)]
cmd = self._init_command(subgroup, singularity_options)
# If name is provided assume referencing an instance
instance_name = self.name
if name is not None:
instance_name = name
cmd = cmd + [instance_name]
# Print verbose output
if not (quiet or self.quiet):
bot.info(" ".join(cmd))
output = run_command(cmd, sudo=sudo, sudo_options=sudo_options, quiet=True)
if output["return_code"] != 0:
message = "%s : return code %s" % (output["message"], output["return_code"])
bot.error(message)
return output["return_code"]
return output["return_code"]
def mkdir_p(path):
'''mkdir_p attempts to get the same functionality as mkdir -p
:param path: the path to create.
'''
try:
os.makedirs(path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
bot.error("Error creating path %s, exiting." % path)
sys.exit(1)
def _run_checks(self):
'''basic sanity checks for the file name (and others if needed) before
attempting parsing.
'''
if self.recipe is not None:
# Does the recipe provided exist?
if not os.path.exists(self.recipe):
bot.error("Cannot find %s, is the path correct?" % self.recipe)
sys.exit(1)
# Ensure we carry fullpath
self.recipe = os.path.abspath(self.recipe)
def generate_bind_list(self, bindlist=None):
"""generate bind string will take a single string or list of binds, and
return a list that can be added to an exec or run command. For example,
the following map as follows:
['/host:/container', '/both'] --> ["--bind", "/host:/container","--bind","/both" ]
['/both'] --> ["--bind", "/both"]
'/host:container' --> ["--bind", "/host:container"]
None --> []
An empty bind or otherwise value of None should return an empty list.
The binds are also checked on the host.
Parameters
==========
bindlist: a string or list of bind mounts
"""
binds = []
# Case 1: No binds provided
if not bindlist:
return binds
# Case 2: provides a long string or non list, and must be split
if not isinstance(bindlist, list):
bindlist = bindlist.split(" ")
for bind in bindlist:
# Still cannot be None
if bind:
bot.debug("Adding bind %s" % bind)
binds += ["--bind", bind]
# Check that exists on host
host = bind.split(":")[0]
if not os.path.exists(host):
bot.error("%s does not exist on host." % bind)
sys.exit(1)
return binds
def stop(self, name=None, sudo=False):
"""stop an instance. This is done by default when an instance is created.
Parameters
==========
name: a name for the instance
sudo: if the user wants to run the command with sudo
USAGE:
singularity [...] instance.stop [...] <instance name>
"""
from spython.utils import check_install, run_command
check_install()
subgroup = "instance.stop"
if "version 3" in self.version():
subgroup = ["instance", "stop"]
cmd = self._init_command(subgroup)
# If name is provided assume referencing an instance
instance_name = self.name
if name is not None:
instance_name = name
cmd = cmd + [instance_name]
output = run_command(cmd, sudo=sudo, quiet=True)
if output["return_code"] != 0:
message = "%s : return code %s" % (output["message"],
output["return_code"])
bot.error(message)
return output["return_code"]
return output["return_code"]
def stopall(self, sudo=False, quiet=True):
'''stop ALL instances. This command is only added to the command group
as it doesn't make sense to call from a single instance
Parameters
==========
sudo: if the command should be done with sudo (exposes different set of
instances)
'''
from spython.utils import run_command, check_install
check_install()
cmd = self._init_command('instance.stop')
cmd = cmd + ['--all']
output = run_command(cmd, sudo=sudo, quiet=quiet)
if output['return_code'] != 0:
message = '%s : return code %s' % (output['message'],
output['return_code'])
bot.error(message)
return output['return_code']
return output['return_code']
def pull(self,
image=None,
name=None,
pull_folder='',
ext="simg",
force=False,
capture=False,
name_by_commit=False,
name_by_hash=False,
stream=False):
'''pull will pull a singularity hub or Docker image
Parameters
==========
image: the complete image uri. If not provided, the client loaded is used
pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to
user specified location instead.
Docker and Singularity Hub Naming
---------------------------------
name: a custom name to use, to override default
ext: if no name specified, the default extension to use.
'''
from spython.utils import check_install
check_install()
cmd = self._init_command('pull')
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
# If it's still None, no go!
if image is None:
bot.exit('You must provide an image uri, or use client.load() first.')
# Singularity Only supports shub and Docker pull
if not re.search('^(shub|docker)://', image):
bot.exit("pull only valid for docker and shub. Use sregistry client.")
# Did the user ask for a custom pull folder?
if pull_folder:
self.setenv('SINGULARITY_PULLFOLDER', pull_folder)
# If we still don't have a custom name, base off of image uri.
# Determine how to tell client to name the image, preference to hash
if name_by_hash is True:
cmd.append('--hash')
elif name_by_commit is True:
cmd.append('--commit')
elif name is None:
name = self._get_filename(image, ext)
# Only add name if we aren't naming by hash or commit
if not name_by_commit and not name_by_hash:
cmd = cmd + ["--name", name]
if force is True:
cmd = cmd + ["--force"]
cmd.append(image)
bot.info(' '.join(cmd))
# If name is still None, make empty string
if name is None:
name = ''
final_image = os.path.join(pull_folder, name)
# Option 1: For hash or commit, need return value to get final_image
if name_by_commit or name_by_hash:
# Set pull to temporary location
tmp_folder = tempfile.mkdtemp()
self.setenv('SINGULARITY_PULLFOLDER', tmp_folder)
self._run_command(cmd, capture=capture)
try:
tmp_image = os.path.join(tmp_folder, os.listdir(tmp_folder)[0])
final_image = os.path.join(pull_folder,
os.path.basename(tmp_image))
shutil.move(tmp_image, final_image)
shutil.rmtree(tmp_folder)
except:
bot.error('Issue pulling image with commit or hash, try without?')
# Option 2: Streaming we just run to show user
elif stream is False:
self._run_command(cmd, capture=capture)
# Option 3: A custom name we can predict (not commit/hash) and can also show
else:
return final_image, stream_command(cmd, sudo=False)
if os.path.exists(final_image):
bot.info(final_image)
return final_image
def _check_install(self):
'''ensure that singularity is installed, and exit if not.
'''
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
def instances(self, name=None, return_json=False, quiet=False):
'''list instances. For Singularity, this is provided as a command sub
group.
singularity instance.list
Return codes provided are different from standard linux:
see https://github.com/singularityware/singularity/issues/1706
Parameters
==========
return_json: return a json list of instances instead of objects (False)
name: if defined, return the list for just one instance (used to ged pid)
Return Code -- Reason
0 -- Instances Found
1 -- No Instances, libexecdir value not found, functions file not found
255 -- Couldn't get UID
'''
from spython.instance.cmd.iutils import parse_table
from spython.utils import check_install
check_install()
subgroup = 'instance.list'
if get_singularity_version().find("version 3"):
subgroup = ["instance", "list"]
cmd = self._init_command(subgroup)
# If the user has provided a name, we want to see a particular instance
if name is not None:
cmd.append(name)
output = run_command(cmd, quiet=True)
instances = None
# Success, we have instances
if output['return_code'] == 0:
# Only print the table if we are returning json
if quiet is False:
print(''.join(output['message']))
# Prepare json result from table
header = ['daemon_name', 'pid', 'container_image']
instances = parse_table(output['message'][0], header)
# Does the user want instance objects instead?
listing = []
if return_json is False:
for i in instances:
new_instance = self.instance(pid=i['pid'],
name=i['daemon_name'],
image=i['container_image'],
start=False)
listing.append(new_instance)
instances = listing
# Couldn't get UID
elif output['return_code'] == 255:
bot.error("Couldn't get UID")
# Return code of 0
else:
bot.info('No instances found.')
# If we are given a name, return just one
if name is not None and instances is not None:
if len(instances) == 1:
instances = instances[0]
return instances
def start(self,
image=None,
name=None,
args=None,
sudo=False,
options=None,
capture=False):
"""start an instance. This is done by default when an instance is created.
Parameters
==========
image: optionally, an image uri (if called as a command from Client)
name: a name for the instance
sudo: if the user wants to run the command with sudo
capture: capture output, default is False. With True likely to hang.
args: arguments to provide to the instance (supported Singularity 3.1+)
options: a list of tuples, each an option to give to the start command
[("--bind", "/tmp"),...]
USAGE:
singularity [...] instance.start [...] <container path> <instance name>
"""
from spython.utils import run_command, check_install
check_install()
# If name provided, over write robot (default)
if name is not None:
self.name = name
# If an image isn't provided, we have an initialized instance
if image is None:
# Not having this means it was called as a command, without an image
if not hasattr(self, "_image"):
bot.exit("Please provide an image, or create an Instance first.")
image = self._image
# Derive subgroup command based on singularity version
subgroup = "instance.start"
if "version 3" in self.version():
subgroup = ["instance", "start"]
cmd = self._init_command(subgroup)
# Add options, if they are provided
if not isinstance(options, list):
options = [] if options is None else options.split(" ")
# Assemble the command!
cmd = cmd + options + [image, self.name]
# If arguments are provided
if args is not None:
if not isinstance(args, list):
args = [args]
cmd = cmd + args
# Save the options and cmd, if the user wants to see them later
self.options = options
self.args = args
self.cmd = cmd
output = run_command(cmd, sudo=sudo, quiet=True, capture=capture)
if output["return_code"] == 0:
self._update_metadata()
else:
message = "%s : return code %s" % (output["message"],
output["return_code"])
bot.error(message)
return self
def list_instances(self,
name=None,
return_json=False,
quiet=False,
sudo=False):
"""list instances. For Singularity, this is provided as a command sub
group.
singularity instance.list
Return codes provided are different from standard linux:
see https://github.com/singularityware/singularity/issues/1706
Parameters
==========
return_json: return a json list of instances instead of objects (False)
name: if defined, return the list for just one instance (used to ged pid)
Return Code -- Reason
0 -- Instances Found
1 -- No Instances, libexecdir value not found, functions file not found
255 -- Couldn't get UID
"""
from spython.instance.cmd.iutils import parse_table
from spython.utils import check_install
check_install()
subgroup = "instance.list"
if "version 3" in self.version():
subgroup = ["instance", "list"]
cmd = self._init_command(subgroup)
# If the user has provided a name, we want to see a particular instance
if name is not None:
cmd.append(name)
output = run_command(cmd, quiet=True, sudo=sudo)
instances = []
# Success, we have instances
if output["return_code"] == 0:
# Only print the table if we are returning json
if not quiet:
print("".join(output["message"]))
# Prepare json result from table
# Singularity after 3.5.2 has an added ipaddress
try:
header = ["daemon_name", "pid", "container_image"]
instances = parse_table(output["message"][0], header)
except:
header = ["daemon_name", "pid", "ip", "container_image"]
instances = parse_table(output["message"][0], header)
# Does the user want instance objects instead?
listing = []
if not return_json:
for i in instances:
# If the user has provided a name, only add instance matches
if name is not None:
if name != i["daemon_name"]:
continue
# Otherwise, add instances to the listing
new_instance = self.instance(
pid=i["pid"],
name=i["daemon_name"],
image=i["container_image"],
start=False,
)
listing.append(new_instance)
instances = listing
# Couldn't get UID
elif output["return_code"] == 255:
bot.error("Couldn't get UID")
# Return code of 0
else:
bot.info("No instances found.")
# If we are given a name, return just one
if name is not None and instances not in [None, []]:
if len(instances) == 1:
instances = instances[0]
return instances
def list_instances(
self,
name=None,
return_json=False,
quiet=False,
sudo=False,
sudo_options=None,
singularity_options=None,
):
"""list instances. For Singularity, this is provided as a command sub
group.
singularity instance list
Return codes provided are different from standard linux:
see https://github.com/singularityware/singularity/issues/1706
Since we expect json output, we don't support older versions of Singularity.
Parameters
==========
return_json: return a json list of instances instead of objects (False)
name: if defined, return the list for just one instance (used to ged pid)
singularity_options: a list of options to provide to the singularity client
Return Code -- Reason
0 -- Instances Found
1 -- No Instances, libexecdir value not found, functions file not found
255 -- Couldn't get UID
"""
from spython.utils import check_install
check_install()
subgroup = ["instance", "list", "--json"]
if "version 3" not in self.version():
bot.exit("This version of Singularity Python does not support < 3.0.")
cmd = self._init_command(subgroup, singularity_options)
# If the user has provided a name, we want to see a particular instance
if name is not None:
cmd.append(name)
# Does the user want to see the command printed?
if not (quiet or self.quiet):
bot.info(" ".join(cmd))
output = run_command(cmd, quiet=True, sudo=sudo, sudo_options=sudo_options)
instances = []
# Success, we have instances
if output["return_code"] == 0:
instances = json.loads(output["message"][0]).get("instances", {})
# Does the user want instance objects instead?
listing = []
if not return_json:
for i in instances:
# If the user has provided a name, only add instance matches
if name is not None:
if name != i["instance"]:
continue
# Otherwise, add instances to the listing
new_instance = self.instance(
pid=i.get("pid"),
ip_address=i.get("ip"),
name=i.get("instance") or i.get("daemon_name"),
log_err_path=i.get("logErrPath"),
log_out_path=i.get("logOutPath"),
image=i.get("img") or i.get("container_image"),
start=False,
)
listing.append(new_instance)
instances = listing
# Couldn't get UID
elif output["return_code"] == 255:
bot.error("Couldn't get UID")
# Return code of 0
else:
bot.info("No instances found.")
# If we are given a name, return just one
if name is not None and instances and len(instances) == 1:
instances = instances[0]
return instances
def execute(self,
image=None,
command=None,
app=None,
writable=False,
contain=False,
bind=None,
stream=False,
nv=False):
''' execute: send a command to a container
Parameters
==========
image: full path to singularity image
command: command to send to container
app: if not None, execute a command in context of an app
writable: This option makes the file system accessible as read/write
contain: This option disables the automatic sharing of writable
filesystems on your host
bind: list or single string of bind paths.
This option allows you to map directories on your host system to
directories within your container using bind mounts
nv: if True, load Nvidia Drivers in runtime (default False)
'''
from spython.utils import check_install
check_install()
cmd = self._init_command('exec')
# nv option leverages any GPU cards
if nv is True:
cmd += ['--nv']
# If the image is given as a list, it's probably the command
if isinstance(image, list):
command = image
image = None
if command is not None:
# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
self.quiet = True
# If an instance is provided, grab it's name
if isinstance(image, self.instance):
image = image.get_uri()
# Does the user want to use bind paths option?
if bind is not None:
cmd += self._generate_bind_list(bind)
# Does the user want to run an app?
if app is not None:
cmd = cmd + ['--app', app]
sudo = False
if writable is True:
sudo = True
if not isinstance(command, list):
command = command.split(' ')
cmd = cmd + [image] + command
if stream is False:
return self._run_command(cmd, sudo=sudo)
return stream_command(cmd, sudo=sudo)
bot.error('Please include a command (list) to execute.')