def get(self, name, return_json=False, quiet=False, singularity_options=None): """get is a list for a single instance. It is assumed to be running, and we need to look up the PID, etc. """ from spython.utils import check_install check_install() # Ensure compatible for singularity prior to 3.0, and after 3.0 subgroup = "instance.list" if "version 3" in self.version(): subgroup = ["instance", "list"] cmd = self._init_command(subgroup, singularity_options) cmd.append(name) output = self.run_command(cmd, quiet=True) # Success, we have instances if output["return_code"] == 0: # Only print the table if we are returning json if not quiet: print("".join(output["message"])) # Prepare json result from table header = ["daemon_name", "pid", "container_image"] instances = parse_table(output["message"][0], header) # Does the user want instance objects instead? listing = [] if not return_json: for i in instances: new_instance = Instance( pid=i["pid"], name=i["daemon_name"], image=i["container_image"], start=False, ) listing.append(new_instance) instances = listing # Couldn't get UID elif output["return_code"] == 255: bot.error("Couldn't get UID") # Return code of 0 else: bot.info("No instances found.") # If we are given a name, return just one if name is not None and len(instances) == 1: instances = instances[0] return instances
def run_command(self, cmd, sudo=False, quiet=False, capture=True): '''run_command is a wrapper for the global run_command, checking first for sudo and exiting on error if needed. The message is returned as a list of lines for the calling function to parse, and stdout uses the parent process so it appears for the user. Parameters ========== cmd: the command to run sudo: does the command require sudo? On success, returns result. Otherwise, exists on error ''' result = run_cmd(cmd, sudo=sudo, capture=capture) message = result['message'] return_code = result['return_code'] if result['return_code'] == 0: if len(message) == 1: message = message[0] return message if quiet is False: bot.error("Return Code %s: %s" %(return_code, message))
def _load_bootstrap(self, line): '''load bootstrap checks that the bootstrap is Docker, otherwise we exit on fail (there is no other option to convert to Dockerfile! ''' if 'docker' not in line.lower(): bot.error('docker not detected as Bootstrap!') sys.exit(1)
def stopall(self, sudo=False, quiet=True): """stop ALL instances. This command is only added to the command group as it doesn't make sense to call from a single instance Parameters ========== sudo: if the command should be done with sudo (exposes different set of instances) """ from spython.utils import check_install check_install() subgroup = "instance.stop" if "version 3" in self.version(): subgroup = ["instance", "stop"] cmd = self._init_command(subgroup) cmd = cmd + ["--all"] output = run_command(cmd, sudo=sudo, quiet=quiet) if output["return_code"] != 0: message = "%s : return code %s" % (output["message"], output["return_code"]) bot.error(message) return output["return_code"] return output["return_code"]
def stop(self, name=None, sudo=False): '''start an instance. This is done by default when an instance is created. Parameters ========== image: optionally, an image uri (if called as a command from Client) name: a name for the instance sudo: if the user wants to run the command with sudo USAGE: singularity [...] instance.start [...] <container path> <instance name> ''' from spython.utils import (check_install, run_command) check_install() cmd = self._init_command('instance.stop') # If name is provided assume referencing an instance instance_name = self.name if name is not None: instance_name = name cmd = cmd + [instance_name] output = run_command(cmd, sudo=sudo, quiet=True) if output['return_code'] != 0: message = '%s : return code %s' % (output['message'], output['return_code']) bot.error(message) return output['return_code'] return output['return_code']
def execute(self, image=None, command=None, app=None, writable=False, contain=False, bind=None): ''' execute: send a command to a container Parameters ========== image: full path to singularity image command: command to send to container app: if not None, execute a command in context of an app writable: This option makes the file system accessible as read/write contain: This option disables the automatic sharing of writable filesystems on your host bind: list or single string of bind paths. This option allows you to map directories on your host system to directories within your container using bind mounts ''' cmd = self._init_command('exec') self.check_install() # If the image is given as a list, it's probably the command if isinstance(image, list): command = image image = None if command is not None: # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() # Does the user want to use bind paths option? if bind is not None: cmd += self._generate_bind_list(bind) # Does the user want to run an app? if app is not None: cmd = cmd + ['--app', app] sudo = False if writable is True: sudo = True if not isinstance(command, list): command = command.split(' ') cmd = cmd + [image] + command return self._run_command(cmd,sudo=sudo) bot.error('Please include a command (list) to execute.')
def start(self, image=None, name=None, sudo=False, options=[], capture=False): '''start an instance. This is done by default when an instance is created. Parameters ========== image: optionally, an image uri (if called as a command from Client) name: a name for the instance sudo: if the user wants to run the command with sudo capture: capture output, default is False. With True likely to hang. options: a list of tuples, each an option to give to the start command [("--bind", "/tmp"),...] USAGE: singularity [...] instance.start [...] <container path> <instance name> ''' from spython.utils import (run_command, check_install) check_install() # If no name provided, give it an excellent one! if name is None: name = self.RobotNamer.generate() self.name = name.replace('-', '_') # If an image isn't provided, we have an initialized instance if image is None: # Not having this means it was called as a command, without an image if not hasattr(self, "_image"): bot.error('Please provide an image, or create an Instance first.') sys.exit(1) image = self._image cmd = self._init_command('instance.start') # Add options, if they are provided if not isinstance(options, list): options = options.split(' ') # Assemble the command! cmd = cmd + options + [image, self.name] # Save the options and cmd, if the user wants to see them later self.options = options self.cmd = cmd output = run_command(cmd, sudo=sudo, quiet=True, capture=capture) if output['return_code'] == 0: self._update_metadata() else: message = '%s : return code %s' % (output['message'], output['return_code']) bot.error(message) return self
def stop( self, name=None, sudo=False, sudo_options=None, timeout=None, singularity_options=None, quiet=True, ): """stop an instance. This is done by default when an instance is created. Parameters ========== name: a name for the instance sudo: if the user wants to run the command with sudo singularity_options: a list of options to provide to the singularity client quiet: Do not print verbose output. timeout: forcebly kill non-stopped instance after the timeout specified in seconds USAGE: singularity [...] instance.stop [...] <instance name> """ from spython.utils import check_install, run_command check_install() subgroup = "instance.stop" if "version 3" in self.version(): subgroup = ["instance", "stop"] if timeout: subgroup += ["-t", str(timeout)] cmd = self._init_command(subgroup, singularity_options) # If name is provided assume referencing an instance instance_name = self.name if name is not None: instance_name = name cmd = cmd + [instance_name] # Print verbose output if not (quiet or self.quiet): bot.info(" ".join(cmd)) output = run_command(cmd, sudo=sudo, sudo_options=sudo_options, quiet=True) if output["return_code"] != 0: message = "%s : return code %s" % (output["message"], output["return_code"]) bot.error(message) return output["return_code"] return output["return_code"]
def mkdir_p(path): '''mkdir_p attempts to get the same functionality as mkdir -p :param path: the path to create. ''' try: os.makedirs(path) except OSError as e: if e.errno == errno.EEXIST and os.path.isdir(path): pass else: bot.error("Error creating path %s, exiting." % path) sys.exit(1)
def _run_checks(self): '''basic sanity checks for the file name (and others if needed) before attempting parsing. ''' if self.recipe is not None: # Does the recipe provided exist? if not os.path.exists(self.recipe): bot.error("Cannot find %s, is the path correct?" % self.recipe) sys.exit(1) # Ensure we carry fullpath self.recipe = os.path.abspath(self.recipe)
def generate_bind_list(self, bindlist=None): """generate bind string will take a single string or list of binds, and return a list that can be added to an exec or run command. For example, the following map as follows: ['/host:/container', '/both'] --> ["--bind", "/host:/container","--bind","/both" ] ['/both'] --> ["--bind", "/both"] '/host:container' --> ["--bind", "/host:container"] None --> [] An empty bind or otherwise value of None should return an empty list. The binds are also checked on the host. Parameters ========== bindlist: a string or list of bind mounts """ binds = [] # Case 1: No binds provided if not bindlist: return binds # Case 2: provides a long string or non list, and must be split if not isinstance(bindlist, list): bindlist = bindlist.split(" ") for bind in bindlist: # Still cannot be None if bind: bot.debug("Adding bind %s" % bind) binds += ["--bind", bind] # Check that exists on host host = bind.split(":")[0] if not os.path.exists(host): bot.error("%s does not exist on host." % bind) sys.exit(1) return binds
def stop(self, name=None, sudo=False): """stop an instance. This is done by default when an instance is created. Parameters ========== name: a name for the instance sudo: if the user wants to run the command with sudo USAGE: singularity [...] instance.stop [...] <instance name> """ from spython.utils import check_install, run_command check_install() subgroup = "instance.stop" if "version 3" in self.version(): subgroup = ["instance", "stop"] cmd = self._init_command(subgroup) # If name is provided assume referencing an instance instance_name = self.name if name is not None: instance_name = name cmd = cmd + [instance_name] output = run_command(cmd, sudo=sudo, quiet=True) if output["return_code"] != 0: message = "%s : return code %s" % (output["message"], output["return_code"]) bot.error(message) return output["return_code"] return output["return_code"]
def stopall(self, sudo=False, quiet=True): '''stop ALL instances. This command is only added to the command group as it doesn't make sense to call from a single instance Parameters ========== sudo: if the command should be done with sudo (exposes different set of instances) ''' from spython.utils import run_command, check_install check_install() cmd = self._init_command('instance.stop') cmd = cmd + ['--all'] output = run_command(cmd, sudo=sudo, quiet=quiet) if output['return_code'] != 0: message = '%s : return code %s' % (output['message'], output['return_code']) bot.error(message) return output['return_code'] return output['return_code']
def pull(self, image=None, name=None, pull_folder='', ext="simg", force=False, capture=False, name_by_commit=False, name_by_hash=False, stream=False): '''pull will pull a singularity hub or Docker image Parameters ========== image: the complete image uri. If not provided, the client loaded is used pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to user specified location instead. Docker and Singularity Hub Naming --------------------------------- name: a custom name to use, to override default ext: if no name specified, the default extension to use. ''' from spython.utils import check_install check_install() cmd = self._init_command('pull') # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() # If it's still None, no go! if image is None: bot.exit('You must provide an image uri, or use client.load() first.') # Singularity Only supports shub and Docker pull if not re.search('^(shub|docker)://', image): bot.exit("pull only valid for docker and shub. Use sregistry client.") # Did the user ask for a custom pull folder? if pull_folder: self.setenv('SINGULARITY_PULLFOLDER', pull_folder) # If we still don't have a custom name, base off of image uri. # Determine how to tell client to name the image, preference to hash if name_by_hash is True: cmd.append('--hash') elif name_by_commit is True: cmd.append('--commit') elif name is None: name = self._get_filename(image, ext) # Only add name if we aren't naming by hash or commit if not name_by_commit and not name_by_hash: cmd = cmd + ["--name", name] if force is True: cmd = cmd + ["--force"] cmd.append(image) bot.info(' '.join(cmd)) # If name is still None, make empty string if name is None: name = '' final_image = os.path.join(pull_folder, name) # Option 1: For hash or commit, need return value to get final_image if name_by_commit or name_by_hash: # Set pull to temporary location tmp_folder = tempfile.mkdtemp() self.setenv('SINGULARITY_PULLFOLDER', tmp_folder) self._run_command(cmd, capture=capture) try: tmp_image = os.path.join(tmp_folder, os.listdir(tmp_folder)[0]) final_image = os.path.join(pull_folder, os.path.basename(tmp_image)) shutil.move(tmp_image, final_image) shutil.rmtree(tmp_folder) except: bot.error('Issue pulling image with commit or hash, try without?') # Option 2: Streaming we just run to show user elif stream is False: self._run_command(cmd, capture=capture) # Option 3: A custom name we can predict (not commit/hash) and can also show else: return final_image, stream_command(cmd, sudo=False) if os.path.exists(final_image): bot.info(final_image) return final_image
def _check_install(self): '''ensure that singularity is installed, and exit if not. ''' if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1)
def instances(self, name=None, return_json=False, quiet=False): '''list instances. For Singularity, this is provided as a command sub group. singularity instance.list Return codes provided are different from standard linux: see https://github.com/singularityware/singularity/issues/1706 Parameters ========== return_json: return a json list of instances instead of objects (False) name: if defined, return the list for just one instance (used to ged pid) Return Code -- Reason 0 -- Instances Found 1 -- No Instances, libexecdir value not found, functions file not found 255 -- Couldn't get UID ''' from spython.instance.cmd.iutils import parse_table from spython.utils import check_install check_install() subgroup = 'instance.list' if get_singularity_version().find("version 3"): subgroup = ["instance", "list"] cmd = self._init_command(subgroup) # If the user has provided a name, we want to see a particular instance if name is not None: cmd.append(name) output = run_command(cmd, quiet=True) instances = None # Success, we have instances if output['return_code'] == 0: # Only print the table if we are returning json if quiet is False: print(''.join(output['message'])) # Prepare json result from table header = ['daemon_name', 'pid', 'container_image'] instances = parse_table(output['message'][0], header) # Does the user want instance objects instead? listing = [] if return_json is False: for i in instances: new_instance = self.instance(pid=i['pid'], name=i['daemon_name'], image=i['container_image'], start=False) listing.append(new_instance) instances = listing # Couldn't get UID elif output['return_code'] == 255: bot.error("Couldn't get UID") # Return code of 0 else: bot.info('No instances found.') # If we are given a name, return just one if name is not None and instances is not None: if len(instances) == 1: instances = instances[0] return instances
def start(self, image=None, name=None, args=None, sudo=False, options=None, capture=False): """start an instance. This is done by default when an instance is created. Parameters ========== image: optionally, an image uri (if called as a command from Client) name: a name for the instance sudo: if the user wants to run the command with sudo capture: capture output, default is False. With True likely to hang. args: arguments to provide to the instance (supported Singularity 3.1+) options: a list of tuples, each an option to give to the start command [("--bind", "/tmp"),...] USAGE: singularity [...] instance.start [...] <container path> <instance name> """ from spython.utils import run_command, check_install check_install() # If name provided, over write robot (default) if name is not None: self.name = name # If an image isn't provided, we have an initialized instance if image is None: # Not having this means it was called as a command, without an image if not hasattr(self, "_image"): bot.exit("Please provide an image, or create an Instance first.") image = self._image # Derive subgroup command based on singularity version subgroup = "instance.start" if "version 3" in self.version(): subgroup = ["instance", "start"] cmd = self._init_command(subgroup) # Add options, if they are provided if not isinstance(options, list): options = [] if options is None else options.split(" ") # Assemble the command! cmd = cmd + options + [image, self.name] # If arguments are provided if args is not None: if not isinstance(args, list): args = [args] cmd = cmd + args # Save the options and cmd, if the user wants to see them later self.options = options self.args = args self.cmd = cmd output = run_command(cmd, sudo=sudo, quiet=True, capture=capture) if output["return_code"] == 0: self._update_metadata() else: message = "%s : return code %s" % (output["message"], output["return_code"]) bot.error(message) return self
def list_instances(self, name=None, return_json=False, quiet=False, sudo=False): """list instances. For Singularity, this is provided as a command sub group. singularity instance.list Return codes provided are different from standard linux: see https://github.com/singularityware/singularity/issues/1706 Parameters ========== return_json: return a json list of instances instead of objects (False) name: if defined, return the list for just one instance (used to ged pid) Return Code -- Reason 0 -- Instances Found 1 -- No Instances, libexecdir value not found, functions file not found 255 -- Couldn't get UID """ from spython.instance.cmd.iutils import parse_table from spython.utils import check_install check_install() subgroup = "instance.list" if "version 3" in self.version(): subgroup = ["instance", "list"] cmd = self._init_command(subgroup) # If the user has provided a name, we want to see a particular instance if name is not None: cmd.append(name) output = run_command(cmd, quiet=True, sudo=sudo) instances = [] # Success, we have instances if output["return_code"] == 0: # Only print the table if we are returning json if not quiet: print("".join(output["message"])) # Prepare json result from table # Singularity after 3.5.2 has an added ipaddress try: header = ["daemon_name", "pid", "container_image"] instances = parse_table(output["message"][0], header) except: header = ["daemon_name", "pid", "ip", "container_image"] instances = parse_table(output["message"][0], header) # Does the user want instance objects instead? listing = [] if not return_json: for i in instances: # If the user has provided a name, only add instance matches if name is not None: if name != i["daemon_name"]: continue # Otherwise, add instances to the listing new_instance = self.instance( pid=i["pid"], name=i["daemon_name"], image=i["container_image"], start=False, ) listing.append(new_instance) instances = listing # Couldn't get UID elif output["return_code"] == 255: bot.error("Couldn't get UID") # Return code of 0 else: bot.info("No instances found.") # If we are given a name, return just one if name is not None and instances not in [None, []]: if len(instances) == 1: instances = instances[0] return instances
def list_instances( self, name=None, return_json=False, quiet=False, sudo=False, sudo_options=None, singularity_options=None, ): """list instances. For Singularity, this is provided as a command sub group. singularity instance list Return codes provided are different from standard linux: see https://github.com/singularityware/singularity/issues/1706 Since we expect json output, we don't support older versions of Singularity. Parameters ========== return_json: return a json list of instances instead of objects (False) name: if defined, return the list for just one instance (used to ged pid) singularity_options: a list of options to provide to the singularity client Return Code -- Reason 0 -- Instances Found 1 -- No Instances, libexecdir value not found, functions file not found 255 -- Couldn't get UID """ from spython.utils import check_install check_install() subgroup = ["instance", "list", "--json"] if "version 3" not in self.version(): bot.exit("This version of Singularity Python does not support < 3.0.") cmd = self._init_command(subgroup, singularity_options) # If the user has provided a name, we want to see a particular instance if name is not None: cmd.append(name) # Does the user want to see the command printed? if not (quiet or self.quiet): bot.info(" ".join(cmd)) output = run_command(cmd, quiet=True, sudo=sudo, sudo_options=sudo_options) instances = [] # Success, we have instances if output["return_code"] == 0: instances = json.loads(output["message"][0]).get("instances", {}) # Does the user want instance objects instead? listing = [] if not return_json: for i in instances: # If the user has provided a name, only add instance matches if name is not None: if name != i["instance"]: continue # Otherwise, add instances to the listing new_instance = self.instance( pid=i.get("pid"), ip_address=i.get("ip"), name=i.get("instance") or i.get("daemon_name"), log_err_path=i.get("logErrPath"), log_out_path=i.get("logOutPath"), image=i.get("img") or i.get("container_image"), start=False, ) listing.append(new_instance) instances = listing # Couldn't get UID elif output["return_code"] == 255: bot.error("Couldn't get UID") # Return code of 0 else: bot.info("No instances found.") # If we are given a name, return just one if name is not None and instances and len(instances) == 1: instances = instances[0] return instances
def execute(self, image=None, command=None, app=None, writable=False, contain=False, bind=None, stream=False, nv=False): ''' execute: send a command to a container Parameters ========== image: full path to singularity image command: command to send to container app: if not None, execute a command in context of an app writable: This option makes the file system accessible as read/write contain: This option disables the automatic sharing of writable filesystems on your host bind: list or single string of bind paths. This option allows you to map directories on your host system to directories within your container using bind mounts nv: if True, load Nvidia Drivers in runtime (default False) ''' from spython.utils import check_install check_install() cmd = self._init_command('exec') # nv option leverages any GPU cards if nv is True: cmd += ['--nv'] # If the image is given as a list, it's probably the command if isinstance(image, list): command = image image = None if command is not None: # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() self.quiet = True # If an instance is provided, grab it's name if isinstance(image, self.instance): image = image.get_uri() # Does the user want to use bind paths option? if bind is not None: cmd += self._generate_bind_list(bind) # Does the user want to run an app? if app is not None: cmd = cmd + ['--app', app] sudo = False if writable is True: sudo = True if not isinstance(command, list): command = command.split(' ') cmd = cmd + [image] + command if stream is False: return self._run_command(cmd, sudo=sudo) return stream_command(cmd, sudo=sudo) bot.error('Please include a command (list) to execute.')