def execute( self, command=None, container_id=None, sudo=False, stream=False, singularity_options=None, ): """execute a command to a container instance based on container_id Parameters ========== container_id: the container_id to delete command: the command to execute to the container singularity_options: a list of options to provide to the singularity client sudo: whether to issue the command with sudo (or not) a container started with sudo will belong to the root user If started by a user, the user needs to control deleting it stream: if True, return an iterate to iterate over results of exec. default is False, will return full output as string. Returns ======= return_code: the return code from the delete command. 0 indicates a successful delete, 255 indicates not. """ sudo = self._get_sudo(sudo) container_id = self.get_container_id(container_id) # singularity oci delete cmd = self._init_command("exec", singularity_options) # Add the container_id cmd.append(container_id) if command is not None: if not isinstance(command, list): command = [command] cmd = cmd + command # Execute the command, return response to user if stream: return stream_command(cmd, sudo=sudo) return self._run_command(cmd, sudo=sudo, quiet=True)
def pull(self, image=None, name=None, pull_folder='', ext="simg", force=False, capture=False, name_by_commit=False, name_by_hash=False, stream=False): '''pull will pull a singularity hub or Docker image Parameters ========== image: the complete image uri. If not provided, the client loaded is used pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to user specified location instead. Docker and Singularity Hub Naming --------------------------------- name: a custom name to use, to override default ext: if no name specified, the default extension to use. ''' from spython.utils import check_install check_install() cmd = self._init_command('pull') # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() # If it's still None, no go! if image is None: bot.exit('You must provide an image uri, or use client.load() first.') # Singularity Only supports shub and Docker pull if not re.search('^(shub|docker)://', image): bot.exit("pull only valid for docker and shub. Use sregistry client.") # Did the user ask for a custom pull folder? if pull_folder: self.setenv('SINGULARITY_PULLFOLDER', pull_folder) # If we still don't have a custom name, base off of image uri. # Determine how to tell client to name the image, preference to hash if name_by_hash is True: cmd.append('--hash') elif name_by_commit is True: cmd.append('--commit') elif name is None: name = self._get_filename(image, ext) # Only add name if we aren't naming by hash or commit if not name_by_commit and not name_by_hash: cmd = cmd + ["--name", name] if force is True: cmd = cmd + ["--force"] cmd.append(image) bot.info(' '.join(cmd)) # If name is still None, make empty string if name is None: name = '' final_image = os.path.join(pull_folder, name) # Option 1: For hash or commit, need return value to get final_image if name_by_commit or name_by_hash: # Set pull to temporary location tmp_folder = tempfile.mkdtemp() self.setenv('SINGULARITY_PULLFOLDER', tmp_folder) self._run_command(cmd, capture=capture) try: tmp_image = os.path.join(tmp_folder, os.listdir(tmp_folder)[0]) final_image = os.path.join(pull_folder, os.path.basename(tmp_image)) shutil.move(tmp_image, final_image) shutil.rmtree(tmp_folder) except: bot.error('Issue pulling image with commit or hash, try without?') # Option 2: Streaming we just run to show user elif stream is False: self._run_command(cmd, capture=capture) # Option 3: A custom name we can predict (not commit/hash) and can also show else: return final_image, stream_command(cmd, sudo=False) if os.path.exists(final_image): bot.info(final_image) return final_image
def run(self, image=None, args=None, app=None, sudo=False, writable=False, contain=False, bind=None, stream=False): ''' run will run the container, with or withour arguments (which should be provided in a list) Parameters ========== image: full path to singularity image args: args to include with the run app: if not None, execute a command in context of an app writable: This option makes the file system accessible as read/write contain: This option disables the automatic sharing of writable filesystems on your host bind: list or single string of bind paths. This option allows you to map directories on your host system to directories within your container using bind mounts stream: if True, return <generator> for the user to run ''' self.check_install() cmd = self._init_command('run') # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() # Does the user want to use bind paths option? if bind is not None: cmd += self._generate_bind_list(bind) # Does the user want to run an app? if app is not None: cmd = cmd + ['--app', app] cmd = cmd + [image] # Conditions for needing sudo if writable is True: sudo = True if args is not None: if not isinstance(args, list): args = args.split(' ') cmd = cmd + args if stream is False: result = self._run_command(cmd, sudo=sudo) else: return stream_command(cmd, sudo=sudo) result = result.strip('\n') try: result = json.loads(result) except: pass return result
def run( self, image=None, args=None, app=None, sudo=False, writable=False, contain=False, bind=None, stream=False, nv=False, options=None, return_result=False, ): """ run will run the container, with or withour arguments (which should be provided in a list) Parameters ========== image: full path to singularity image args: args to include with the run app: if not None, execute a command in context of an app writable: This option makes the file system accessible as read/write options: an optional list of options to provide to run. contain: This option disables the automatic sharing of writable filesystems on your host bind: list or single string of bind paths. This option allows you to map directories on your host system to directories within your container using bind mounts stream: if True, return <generator> for the user to run nv: if True, load Nvidia Drivers in runtime (default False) return_result: if True, return entire json object with return code and message result (default is False) """ from spython.utils import check_install check_install() cmd = self._init_command("run") # nv option leverages any GPU cards if nv: cmd += ["--nv"] # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() # If an instance is provided, grab it's name if isinstance(image, self.instance): image = image.get_uri() # If image is still None, not defined by user or previously with client if image is None: bot.exit("Please load or provide an image.") # Does the user want to use bind paths option? if bind is not None: cmd += self._generate_bind_list(bind) # Does the user want to run an app? if app is not None: cmd = cmd + ["--app", app] # Does the user want writable? if writable: cmd.append("--writable") # Add options if options is not None: cmd = cmd + options cmd = cmd + [image] if args is not None: if not isinstance(args, list): args = args.split(" ") cmd = cmd + args if not stream: result = self._run_command(cmd, sudo=sudo, return_result=return_result) else: return stream_command(cmd, sudo=sudo) # If the user wants the raw result object if return_result: return result # Otherwise, we parse the result if it was successful if result: result = result.strip("\n") try: result = json.loads(result) except: pass return result
def build(self, recipe=None, image=None, isolated=False, sandbox=False, writable=False, build_folder=None, robot_name=False, ext='sif', sudo=True, stream=False, force=False, options=None, quiet=False, return_result=False): '''build a singularity image, optionally for an isolated build (requires sudo). If you specify to stream, expect the image name and an iterator to be returned. image, builder = Client.build(...) Parameters ========== recipe: the path to the recipe file (or source to build from). If not defined, we look for "Singularity" file in $PWD image: the image to build (if None, will use arbitary name isolated: if True, run build with --isolated flag sandbox: if True, create a writable sandbox writable: if True, use writable ext3 (sandbox takes preference) build_folder: where the container should be built. ext: the image extension to use. robot_name: boolean, default False. if you don't give your image a name (with "image") then a fun robot name will be generated instead. Highly recommended :) sudo: give sudo to the command (or not) default is True for build options: for all other options, specify them in this list. quiet: quiet verbose printing from the client. return_result: if True, return complete error code / message dictionary ''' from spython.utils import check_install check_install() cmd = self._init_command('build') # If no extra options if not options: options = [] if 'version 3' in self.version(): ext = 'sif' # Force the build if the image / sandbox exists if force: cmd.append('--force') # No image provided, default to use the client's loaded image if recipe is None: recipe = self._get_uri() # If it's still None, try default build recipe if recipe is None: recipe = 'Singularity' if not os.path.exists(recipe): bot.exit('Cannot find %s, exiting.' % image) if image is None: if re.search('(docker|shub)://', recipe) and not robot_name: image = self._get_filename(recipe, ext) else: image = "%s.%s" % (self.RobotNamer.generate(), ext) # Does the user want a custom build folder? if build_folder is not None: if not os.path.exists(build_folder): bot.exit('%s does not exist!' % build_folder) image = os.path.join(build_folder, image) # The user wants to run an isolated build if isolated: cmd.append('--isolated') if sandbox: cmd.append('--sandbox') elif writable: cmd.append('--writable') cmd = cmd + options + [image, recipe] if not stream: self._run_command(cmd, sudo=sudo, quiet=quiet, return_result=return_result, capture=False) else: # Here we return the expected image, and an iterator! # The caller must iterate over return image, stream_command(cmd, sudo=sudo) if os.path.exists(image): return image
def build(self, recipe=None, image=None, isolated=False, sandbox=False, writable=False, build_folder=None, robot_name=False, ext='simg', sudo=True, stream=False): '''build a singularity image, optionally for an isolated build (requires sudo). If you specify to stream, expect the image name and an iterator to be returned. image, builder = Client.build(...) Parameters ========== recipe: the path to the recipe file (or source to build from). If not defined, we look for "Singularity" file in $PWD image: the image to build (if None, will use arbitary name isolated: if True, run build with --isolated flag sandbox: if True, create a writable sandbox writable: if True, use writable ext3 (sandbox takes preference) build_folder: where the container should be built. ext: the image extension to use. robot_name: boolean, default False. if you don't give your image a name (with "image") then a fun robot name will be generated instead. Highly recommended :) sudo: give sudo to the command (or not) default is True for build ''' self.check_install() cmd = self._init_command('build') # No image provided, default to use the client's loaded image if recipe is None: recipe = self._get_uri() # If it's still None, try default build recipe if recipe is None: recipe = 'Singularity' if not os.path.exists(recipe): bot.exit('Cannot find %s, exiting.' % image) if image is None: if re.search('(docker|shub)://', recipe) and robot_name is False: image = self._get_filename(recipe, ext) else: image = "%s.%s" % (self.RobotNamer.generate(), ext) # Does the user want a custom build folder? if build_folder is not None: if not os.path.exists(build_folder): bot.exit('%s does not exist!' % build_folder) image = "%s/%s" % (build_folder, image) if isolated is True: cmd.append('--isolated') if sandbox is True: cmd.append('--sandbox') elif sandbox is True: cmd.append('--writable') cmd = cmd + [image, recipe] if stream is False: output = self._run_command(cmd, sudo=sudo, capture=False) else: # Here we return the expected image, and an iterator! # The caller must iterate over return image, stream_command(cmd, sudo=sudo) if os.path.exists(image): return image
def pull(self, image=None, name=None, pull_folder='', ext="simg", force=False, capture=False, stream=False): '''pull will pull a singularity hub or Docker image Parameters ========== image: the complete image uri. If not provided, the client loaded is used pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to user specified location instead. Docker and Singularity Hub Naming --------------------------------- name: a custom name to use, to override default ext: if no name specified, the default extension to use. ''' from spython.utils import check_install check_install() cmd = self._init_command('pull') # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() # If it's still None, no go! if image is None: bot.exit('You must provide an image uri, or use client.load() first.') # Singularity Only supports shub and Docker pull if not re.search('^(shub|docker)://', image): bot.exit("pull only valid for docker and shub. Use sregistry client.") # Did the user ask for a custom pull folder? if pull_folder: self.setenv('SINGULARITY_PULLFOLDER', pull_folder) # If we still don't have a custom name, base off of image uri. if name is None: name = self._get_filename(image, ext) cmd = cmd + ["--name", name] if force is True: cmd = cmd + ["--force"] cmd.append(image) bot.info(' '.join(cmd)) final_image = os.path.join(pull_folder, name) if stream is False: self._run_command(cmd, capture=capture) else: return final_image, stream_command(cmd, sudo=False) if os.path.exists(final_image): bot.info(final_image) return final_image
def pull(self, image=None, name=None, pull_folder='', ext=None, force=False, capture=False, stream=False, quiet=False): '''pull will pull a singularity hub or Docker image Parameters ========== image: the complete image uri. If not provided, the client loaded is used pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to user specified location instead. Docker and Singularity Hub Naming --------------------------------- name: a custom name to use, to override default ext: if no name specified, the default extension to use. ''' from spython.utils import check_install check_install() cmd = self._init_command('pull') # Quiet is honored if set by the client, or user quiet = quiet or self.quiet if not ext: ext = 'sif' if 'version 3' in self.version() else 'simg' # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() # If it's still None, no go! if image is None: bot.exit('You must provide an image uri, or use client.load() first.') # Singularity Only supports shub and Docker pull if not re.search('^(shub|docker)://', image): bot.exit("pull only valid for docker and shub. Use sregistry client.") # If we still don't have a custom name, base off of image uri. if name is None: name = self._get_filename(image, ext) if pull_folder: final_image = os.path.join(pull_folder, os.path.basename(name)) # Regression Singularity 3.* onward, PULLFOLDER not honored # https://github.com/sylabs/singularity/issues/2788 if 'version 3' in self.version(): name = final_image pull_folder = None # Don't use pull_folder else: final_image = name cmd = cmd + ["--name", name] if force: cmd = cmd + ["--force"] cmd.append(image) if not quiet: bot.info(' '.join(cmd)) with ScopedEnvVar('SINGULARITY_PULLFOLDER', pull_folder): # Option 1: Streaming we just run to show user if not stream: self._run_command(cmd, capture=capture, quiet=quiet) # Option 3: A custom name we can predict (not commit/hash) and can also show else: return final_image, stream_command(cmd, sudo=False) if os.path.exists(final_image) and not quiet: bot.info(final_image) return final_image
def execute(self, image = None, command = None, app = None, writable = False, contain = False, bind = None, stream = False): ''' execute: send a command to a container Parameters ========== image: full path to singularity image command: command to send to container app: if not None, execute a command in context of an app writable: This option makes the file system accessible as read/write contain: This option disables the automatic sharing of writable filesystems on your host bind: list or single string of bind paths. This option allows you to map directories on your host system to directories within your container using bind mounts ''' cmd = self._init_command('exec') self.check_install() # If the image is given as a list, it's probably the command if isinstance(image, list): command = image image = None if command is not None: # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() # Does the user want to use bind paths option? if bind is not None: cmd += self._generate_bind_list(bind) # Does the user want to run an app? if app is not None: cmd = cmd + ['--app', app] sudo = False if writable is True: sudo = True if not isinstance(command, list): command = command.split(' ') cmd = cmd + [image] + command if stream is False: return self._run_command(cmd,sudo=sudo) return stream_command(cmd, sudo=sudo) bot.error('Please include a command (list) to execute.')
def execute( self, image=None, command=None, app=None, writable=False, contain=False, bind=None, stream=False, nv=False, return_result=False, options=None, singularity_options=None, sudo=False, sudo_options=None, quiet=True, environ=None, ): """execute: send a command to a container Parameters ========== image: full path to singularity image command: command to send to container app: if not None, execute a command in context of an app writable: This option makes the file system accessible as read/write contain: This option disables the automatic sharing of writable filesystems on your host options: an optional list of options to provide to execute. singularity_options: a list of options to provide to the singularity client bind: list or single string of bind paths. This option allows you to map directories on your host system to directories within your container using bind mounts nv: if True, load Nvidia Drivers in runtime (default False) return_result: if True, return entire json object with return code and message result not (default) quiet: Do not print verbose output. environ: extra environment to add. """ from spython.utils import check_install check_install() cmd = self._init_command("exec", singularity_options) # nv option leverages any GPU cards if nv: cmd += ["--nv"] # If the image is given as a list, it's probably the command if isinstance(image, list): command = image image = None if command is not None: # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() self.quiet = True # If an instance is provided, grab it's name if isinstance(image, self.instance): image = image.get_uri() # If image is still None, not defined by user or previously with client if image is None: bot.exit("Please load or provide an image.") # Does the user want to use bind paths option? if bind is not None: cmd += self._generate_bind_list(bind) # Does the user want to run an app? if app is not None: cmd = cmd + ["--app", app] if writable: cmd.append("--writable") # Add additional options if options is not None: cmd = cmd + options if not isinstance(command, list): command = command.split(" ") cmd = cmd + [image] + command # Does the user want to see the command printed? if not (quiet or self.quiet): bot.info(" ".join(cmd)) if not stream: return self._run_command( cmd, sudo=sudo, sudo_options=sudo_options, return_result=return_result, quiet=quiet, environ=environ, ) return stream_command(cmd, sudo=sudo, sudo_options=sudo_options) bot.exit("Please include a command (list) to execute.")
def build( self, recipe=None, image=None, isolated=False, sandbox=False, writable=False, build_folder=None, robot_name=False, ext="sif", sudo=True, stream=False, force=False, options=None, quiet=False, return_result=False, sudo_options=None, singularity_options=None, ): """build a singularity image, optionally for an isolated build (requires sudo). If you specify to stream, expect the image name and an iterator to be returned. image, builder = Client.build(...) Parameters ========== recipe: the path to the recipe file (or source to build from). If not defined, we look for "Singularity" file in $PWD image: the image to build (if None, will use arbitary name isolated: if True, run build with --isolated flag sandbox: if True, create a writable sandbox writable: if True, use writable ext3 (sandbox takes preference) build_folder: where the container should be built. ext: the image extension to use. robot_name: boolean, default False. if you don't give your image a name (with "image") then a fun robot name will be generated instead. Highly recommended :) sudo: give sudo to the command (or not) default is True for build sudo_options: options to pass to sudo (e.g. --preserve-env=SINGULARITY_CACHEDIR,SINGULARITY_TMPDIR) options: for all other options, specify them in this list. singularity_options: a list of options to provide to the singularity client quiet: quiet verbose printing from the client. return_result: if True, return complete error code / message dictionary """ from spython.utils import check_install check_install() cmd = self._init_command("build", singularity_options) # If no extra options options = options or [] if "version 3" in self.version(): ext = "sif" # Force the build if the image / sandbox exists if force: cmd.append("--force") # No image provided, default to use the client's loaded image if recipe is None: recipe = self._get_uri() # If it's still None, try default build recipe if recipe is None: recipe = "Singularity" if not os.path.exists(recipe): bot.exit("Cannot find %s, exiting." % image) if image is None: if re.search("(docker|shub|library)://", recipe) and not robot_name: image = self._get_filename(recipe, ext) else: image = "%s.%s" % (self.RobotNamer.generate(), ext) # Does the user want a custom build folder? if build_folder is not None: if not os.path.exists(build_folder): bot.exit("%s does not exist!" % build_folder) image = os.path.join(build_folder, image) # The user wants to run an isolated build if isolated: cmd.append("--isolated") if sandbox: cmd.append("--sandbox") elif writable: cmd.append("--writable") cmd = cmd + options + [image, recipe] # Does the user want to see the command printed? if not (quiet or self.quiet): bot.info(" ".join(cmd)) if not stream: self._run_command( cmd, sudo=sudo, sudo_options=sudo_options, quiet=quiet, return_result=return_result, capture=False, ) else: # Here we return the expected image, and an iterator! # The caller must iterate over return image, stream_command(cmd, sudo=sudo, sudo_options=sudo_options) if os.path.exists(image): return image