def run_post(self): '''run post create commands. Can be added to an instance definition either to run a command directly, or execute a script. The path is assumed to be on the host. post: command: ["mkdir", "-p", "./images/_upload/{0..9}"] OR post: command: "mkdir -p ./images/_upload/{0..9}" ''' if "post" in self.params: if "command" in self.params['post']: command = self.params['post']['command'] # Command must be a list if not isinstance(command, list): command = shlex.split(command) # Capture the return code response = self.client._run_command(command, quiet=True, return_result=True) # If debug on, show output bot.debug("".join(response['message'])) # Alert the user if there is an error if response['return_code'] != 0: bot.error("".join(response['message'])) bot.exit("Return code %s, exiting." % response['return_code'])
def load(self): """load a singularity-compose.yml recipe, and validate it. """ if not os.path.exists(self.filename): bot.error("%s does not exist." % self.filename) sys.exit(1) try: self.config = read_yaml(self.filename, quiet=True) except: # ParserError bot.exit("Cannot parse %s, invalid yaml." % self.filename)
def create(self, ip_address=None, sudo=False, writable_tmpfs=False): """create an instance, if it doesn't exist. """ image = self.get_image() # Case 1: No build context or image defined if image is None: bot.exit( "Please define an image or build context for instance %s" % self.name) # Case 2: Image not built. if not os.path.exists(image): bot.exit("Image %s not found, please run build first." % image) # Finally, create the instance if not self.exists(): bot.info("Creating %s" % self.name) # Command options options = [] # Volumes options += self._get_bind_commands() if sudo: # Ports options += self._get_network_commands(ip_address) # Hostname options += ["--hostname", self.name] # Writable Temporary Directory if writable_tmpfs: options += ["--writable-tmpfs"] # Show the command to the user commands = "%s %s %s %s" % ( " ".join(options), image, self.name, self.args, ) bot.debug("singularity instance start %s" % commands) self.instance = self.client.instance( name=self.name, sudo=self.sudo, options=options, image=image, args=self.args, )
def shell(self, name): """if an instance exists, shell into it. Parameters ========== name: the name of the instance to shell into """ instance = self.get_instance(name) if not instance: bot.exit("Cannot find %s, is it up?" % name) if instance.exists(): self.client.shell(instance.instance.get_uri(), sudo=self.sudo)
def iter_instances(self, names): """yield instances one at a time. If an invalid name is given, exit with error. Parameters ========== names: the names of instances to yield. Must be valid """ # Used to validate instance names instance_names = self.get_instance_names() for name in names: if name not in instance_names: bot.exit("%s is not a valid section name." % name) yield self.instances.get(name)
def set_volumes_from(self, instances): '''volumes from is called after all instances are read in, and then volumes can be mapped (and shared) with both containers. with Docker, this is done with isolation, but for Singularity we will try sharing a bind on the host. Parameters ========== instances: a list of other instances to get volumes from ''' for name in self._volumes_from: if name not in instances: bot.exit('%s not in config is specified to get volumes from.' % name) for volume in instances[name].volumes: if volume not in self.volumes: self.volumes.append(volume)
def run(self, name): """if an instance exists, run it. Parameters ========== name: the name of the instance to run """ instance = self.get_instance(name) if not instance: bot.exit("Cannot find %s, is it up?" % name) if instance.exists(): self.client.quiet = True result = self.client.run(instance.instance.get_uri(), sudo=self.sudo, return_result=True) if result["return_code"] != 0: bot.exit("Return code %s" % result["return_code"]) print("".join([x for x in result["message"] if x]))
def _get_bind_commands(self): '''take a list of volumes, and return the bind commands for Singularity ''' binds = [] for volume in self.volumes: src, dest = volume.split(':') # First try, assume file in root folder if not os.path.exists(os.path.abspath(src)): if os.path.exists(os.path.join(self.working_dir, src)): src = os.path.join(self.working_dir, src) elif os.path.exists( os.path.join(self.working_dir, self.name, src)): src = os.path.join(self.working_dir, self.name, src) else: bot.exit('bind source file %s does not exist' % src) # For the src, ensure that it exists bind = "%s:%s" % (os.path.abspath(src), os.path.abspath(dest)) binds += ['--bind', bind] return binds
def execute(self, name, commands): '''if an instance exists, execute a command to it. Parameters ========== name: the name of the instance to exec to commands: a list of commands to issue ''' instance = self.get_instance(name) if not instance: bot.exit('Cannot find %s, is it up?' % name) if instance.exists(): try: for line in self.client.execute(instance.instance.get_uri(), command=commands, stream=True, sudo=self.sudo): print(line, end='') except subprocess.CalledProcessError: bot.exit('Command had non zero exit status.')
def set_context(self, params): '''set and validate parameters from the singularity-compose.yml, including build (context and recipe). We don't pull or create anything here, but rather just validate that the sections are provided and files exist. ''' # build the container on the host from a context if "build" in params: if "context" not in params['build']: bot.exit("build.context section missing for %s" % self.name) # The user provided a build context self.context = params['build']['context'] # The context folder must exist if not os.path.exists(self.context): bot.exit("build.context %s does not exist." % self.context) self.recipe = params['build'].get('recipe', 'Singularity') # The recipe must exist in the context folder if not os.path.exists(os.path.join(self.context, self.recipe)): bot.exit("%s does not exist in %s" % (self.recipe, self.context)) # An image can be pulled instead elif "image" in params: # If going to pull an image, the context is a folder of same name self.context = self.name # Image is validated when it needs to be used / pulled self.image = params['image'] # We are required to have build OR image else: bot.exit("build or image must be defined for %s" % self.name)
def start(): """main is the entrypoint to singularity compose. We figure out the sub parser based on the command, and then import and call the appropriate main. """ parser = get_parser() def show_help(return_code=0): """print help, including the software version and exit with return code """ version = scompose.__version__ print("\nSingularity Compose v%s" % version) parser.print_help() sys.exit(return_code) # If the user didn't provide any arguments, show the full help if len(sys.argv) == 1: show_help() try: args, extra = parser.parse_known_args() except: sys.exit(0) if args.debug is True: os.environ["MESSAGELEVEL"] = "DEBUG" else: os.environ["MESSAGELEVEL"] = args.log_level # Import the logger to grab verbosity level from scompose.logger import bot # Show the version and exit if args.command == "version" or args.version is True: print(scompose.__version__) sys.exit(0) # Does the user want a shell? if args.command == "build": from .build import main elif args.command == "create": from .create import main elif args.command == "config": from .config import main elif args.command == "down": from .down import main elif args.command == "exec": from .exec import main elif args.command == "logs": from .logs import main elif args.command == "ps": from .ps import main elif args.command == "restart": from .restart import main elif args.command == "run": from .run import main elif args.command == "shell": from .shell import main elif args.command == "up": from .up import main # Pass on to the correct parser return_code = 0 try: main(args=args, parser=parser, extra=extra) sys.exit(return_code) except KeyboardInterrupt: bot.exit("Aborting.") except UnboundLocalError: return_code = 1 sys.exit(return_code)
def build(self, working_dir): '''build an image if called for based on having a recipe and context. Otherwise, pull a container uri to the instance workspace. ''' sif_binary = self.get_image() # If the final image already exists, don't continue if os.path.exists(sif_binary): return # Case 1: Given an image if self.image is not None: if not os.path.exists(self.image): # Can we pull it? if re.search('(docker|library|shub|http?s)[://]', self.image): bot.info('Pulling %s' % self.image) self.client.pull(self.image, name=sif_binary) else: bot.exit('%s is an invalid unique resource identifier.' % self.image) # Case 2: Given a recipe elif self.recipe is not None: # Change directory to the context context = os.path.abspath(self.context) os.chdir(context) # The recipe is expected to exist in the context folder if not os.path.exists(self.recipe): bot.exit('%s not found for build' % self.recipe) # This will likely require sudo, unless --remote or --fakeroot in options try: options = self.get_build_options() # If remote or fakeroot included, don't need sudo sudo = not ("--fakeroot" in options or "--remote" in options) bot.info('Building %s' % self.name) _, stream = self.client.build(image=sif_binary, recipe=self.recipe, options=options, sudo=sudo, stream=True) for line in stream: print(line) except: build = "sudo singularity build %s %s" % ( os.path.basename(sif_binary), self.recipe) bot.warning("Issue building container, try: %s" % build) # Change back to provided working directory os.chdir(working_dir) else: bot.exit("neither image and build defined for %s" % self.name)
def _create( self, names, command="create", writable_tmpfs=True, bridge="10.22.0.0/16", no_resolv=False, ): """create one or more instances. "Command" determines the sub function to call for the instance, which should be "create" or "up". If the user provide a list of names, use them, otherwise default to all instances. Parameters ========== names: the instance names to create command: one of "create" or "up" writable_tmpfs: if the instances should be given writable to tmp bridge: the bridge ip address to use for networking, and generating addresses for the individual containers. see /usr/local/etc/singularity/network/00_bridge.conflist no_resolv: if True, don't create and bind a resolv.conf with Google nameservers. """ # If no names provided, we create all names = names or self.get_instance_names() # Keep track of created instances to determine if we have circular dependency structure created = [] circular_dep = False # Generate ip addresses for each lookup = self.get_ip_lookup(names, bridge) if self.sudo and not no_resolv: # Generate shared hosts file hosts_file = self.create_hosts(lookup) for instance in self.iter_instances(names): depends_on = instance.params.get("depends_on", []) for dep in depends_on: if dep not in created: circular_dep = True # Generate a resolv.conf to bind to the container if self.sudo and not no_resolv: resolv = self.generate_resolv_conf() instance.volumes.append("%s:/etc/resolv.conf" % resolv) # Create a hosts file for the instance based, add as volume instance.volumes.append("%s:/etc/hosts" % hosts_file) # If we get here, execute command and add to list create_func = getattr(instance, command) create_func( working_dir=self.working_dir, writable_tmpfs=writable_tmpfs, ip_address=lookup[instance.name], ) created.append(instance.name) # Run post create commands instance.run_post() if circular_dep: bot.exit( "Unable to create all instances, possible circular dependency." )
def _create(self, names, command="create", writable_tmpfs=True, bridge="10.22.0.0/16", no_resolv=False): '''create one or more instances. "Command" determines the sub function to call for the instance, which should be "create" or "up". If the user provide a list of names, use them, otherwise default to all instances. Parameters ========== names: the instance names to create command: one of "create" or "up" writable_tmpfs: if the instances should be given writable to tmp bridge: the bridge ip address to use for networking, and generating addresses for the individual containers. see /usr/local/etc/singularity/network/00_bridge.conflist no_resolv: if True, don't create and bind a resolv.conf with Google nameservers. ''' # If no names provided, we create all names = names or self.get_instance_names() # Keep a count to determine if we have circular dependency structure created = [] count = 0 # Generate ip addresses for each lookup = self.get_ip_lookup(names, bridge) # Generate shared hosts file hosts_file = self.create_hosts(lookup) # First create those with no dependencies while names: for instance in self.iter_instances(names): # Flag to indicated create do_create = True # Ensure created, skip over if not depends_on = instance.params.get('depends_on', []) for depends_on in depends_on: if depends_on not in created: count += 1 do_create = False if do_create: # Generate a resolv.conf to bind to the container if not no_resolv: resolv = self.generate_resolv_conf() instance.volumes.append('%s:/etc/resolv.conf' % resolv) # Create a hosts file for the instance based, add as volume instance.volumes.append('%s:/etc/hosts' % hosts_file) # If we get here, execute command and add to list create_func = getattr(instance, command) create_func(working_dir=self.working_dir, writable_tmpfs=writable_tmpfs, ip_address=lookup[instance.name]) created.append(instance.name) names.remove(instance.name) # Run post create commands instance.run_post() # Possibly circular dependencies if count >= 100: bot.exit( 'Unable to create all instances, possible circular dependency.' )