def run_post(self):
'''run post create commands. Can be added to an instance definition
either to run a command directly, or execute a script. The path
is assumed to be on the host.
post:
command: ["mkdir", "-p", "./images/_upload/{0..9}"]
OR
post:
command: "mkdir -p ./images/_upload/{0..9}"
'''
if "post" in self.params:
if "command" in self.params['post']:
command = self.params['post']['command']
# Command must be a list
if not isinstance(command, list):
command = shlex.split(command)
# Capture the return code
response = self.client._run_command(command,
quiet=True,
return_result=True)
# If debug on, show output
bot.debug("".join(response['message']))
# Alert the user if there is an error
if response['return_code'] != 0:
bot.error("".join(response['message']))
bot.exit("Return code %s, exiting." %
response['return_code'])
def load(self):
"""load a singularity-compose.yml recipe, and validate it.
"""
if not os.path.exists(self.filename):
bot.error("%s does not exist." % self.filename)
sys.exit(1)
try:
self.config = read_yaml(self.filename, quiet=True)
except: # ParserError
bot.exit("Cannot parse %s, invalid yaml." % self.filename)
def create(self, ip_address=None, sudo=False, writable_tmpfs=False):
"""create an instance, if it doesn't exist.
"""
image = self.get_image()
# Case 1: No build context or image defined
if image is None:
bot.exit(
"Please define an image or build context for instance %s" %
self.name)
# Case 2: Image not built.
if not os.path.exists(image):
bot.exit("Image %s not found, please run build first." % image)
# Finally, create the instance
if not self.exists():
bot.info("Creating %s" % self.name)
# Command options
options = []
# Volumes
options += self._get_bind_commands()
if sudo:
# Ports
options += self._get_network_commands(ip_address)
# Hostname
options += ["--hostname", self.name]
# Writable Temporary Directory
if writable_tmpfs:
options += ["--writable-tmpfs"]
# Show the command to the user
commands = "%s %s %s %s" % (
" ".join(options),
image,
self.name,
self.args,
)
bot.debug("singularity instance start %s" % commands)
self.instance = self.client.instance(
name=self.name,
sudo=self.sudo,
options=options,
image=image,
args=self.args,
)
def shell(self, name):
"""if an instance exists, shell into it.
Parameters
==========
name: the name of the instance to shell into
"""
instance = self.get_instance(name)
if not instance:
bot.exit("Cannot find %s, is it up?" % name)
if instance.exists():
self.client.shell(instance.instance.get_uri(), sudo=self.sudo)
def iter_instances(self, names):
"""yield instances one at a time. If an invalid name is given,
exit with error.
Parameters
==========
names: the names of instances to yield. Must be valid
"""
# Used to validate instance names
instance_names = self.get_instance_names()
for name in names:
if name not in instance_names:
bot.exit("%s is not a valid section name." % name)
yield self.instances.get(name)
def set_volumes_from(self, instances):
'''volumes from is called after all instances are read in, and
then volumes can be mapped (and shared) with both containers.
with Docker, this is done with isolation, but for Singularity
we will try sharing a bind on the host.
Parameters
==========
instances: a list of other instances to get volumes from
'''
for name in self._volumes_from:
if name not in instances:
bot.exit('%s not in config is specified to get volumes from.' %
name)
for volume in instances[name].volumes:
if volume not in self.volumes:
self.volumes.append(volume)
def run(self, name):
"""if an instance exists, run it.
Parameters
==========
name: the name of the instance to run
"""
instance = self.get_instance(name)
if not instance:
bot.exit("Cannot find %s, is it up?" % name)
if instance.exists():
self.client.quiet = True
result = self.client.run(instance.instance.get_uri(),
sudo=self.sudo,
return_result=True)
if result["return_code"] != 0:
bot.exit("Return code %s" % result["return_code"])
print("".join([x for x in result["message"] if x]))
def _get_bind_commands(self):
'''take a list of volumes, and return the bind commands for Singularity
'''
binds = []
for volume in self.volumes:
src, dest = volume.split(':')
# First try, assume file in root folder
if not os.path.exists(os.path.abspath(src)):
if os.path.exists(os.path.join(self.working_dir, src)):
src = os.path.join(self.working_dir, src)
elif os.path.exists(
os.path.join(self.working_dir, self.name, src)):
src = os.path.join(self.working_dir, self.name, src)
else:
bot.exit('bind source file %s does not exist' % src)
# For the src, ensure that it exists
bind = "%s:%s" % (os.path.abspath(src), os.path.abspath(dest))
binds += ['--bind', bind]
return binds
def execute(self, name, commands):
'''if an instance exists, execute a command to it.
Parameters
==========
name: the name of the instance to exec to
commands: a list of commands to issue
'''
instance = self.get_instance(name)
if not instance:
bot.exit('Cannot find %s, is it up?' % name)
if instance.exists():
try:
for line in self.client.execute(instance.instance.get_uri(),
command=commands,
stream=True,
sudo=self.sudo):
print(line, end='')
except subprocess.CalledProcessError:
bot.exit('Command had non zero exit status.')
def set_context(self, params):
'''set and validate parameters from the singularity-compose.yml,
including build (context and recipe). We don't pull or create
anything here, but rather just validate that the sections
are provided and files exist.
'''
# build the container on the host from a context
if "build" in params:
if "context" not in params['build']:
bot.exit("build.context section missing for %s" % self.name)
# The user provided a build context
self.context = params['build']['context']
# The context folder must exist
if not os.path.exists(self.context):
bot.exit("build.context %s does not exist." % self.context)
self.recipe = params['build'].get('recipe', 'Singularity')
# The recipe must exist in the context folder
if not os.path.exists(os.path.join(self.context, self.recipe)):
bot.exit("%s does not exist in %s" %
(self.recipe, self.context))
# An image can be pulled instead
elif "image" in params:
# If going to pull an image, the context is a folder of same name
self.context = self.name
# Image is validated when it needs to be used / pulled
self.image = params['image']
# We are required to have build OR image
else:
bot.exit("build or image must be defined for %s" % self.name)
def start():
"""main is the entrypoint to singularity compose. We figure out the sub
parser based on the command, and then import and call the appropriate
main.
"""
parser = get_parser()
def show_help(return_code=0):
"""print help, including the software version and exit with return code
"""
version = scompose.__version__
print("\nSingularity Compose v%s" % version)
parser.print_help()
sys.exit(return_code)
# If the user didn't provide any arguments, show the full help
if len(sys.argv) == 1:
show_help()
try:
args, extra = parser.parse_known_args()
except:
sys.exit(0)
if args.debug is True:
os.environ["MESSAGELEVEL"] = "DEBUG"
else:
os.environ["MESSAGELEVEL"] = args.log_level
# Import the logger to grab verbosity level
from scompose.logger import bot
# Show the version and exit
if args.command == "version" or args.version is True:
print(scompose.__version__)
sys.exit(0)
# Does the user want a shell?
if args.command == "build":
from .build import main
elif args.command == "create":
from .create import main
elif args.command == "config":
from .config import main
elif args.command == "down":
from .down import main
elif args.command == "exec":
from .exec import main
elif args.command == "logs":
from .logs import main
elif args.command == "ps":
from .ps import main
elif args.command == "restart":
from .restart import main
elif args.command == "run":
from .run import main
elif args.command == "shell":
from .shell import main
elif args.command == "up":
from .up import main
# Pass on to the correct parser
return_code = 0
try:
main(args=args, parser=parser, extra=extra)
sys.exit(return_code)
except KeyboardInterrupt:
bot.exit("Aborting.")
except UnboundLocalError:
return_code = 1
sys.exit(return_code)
def build(self, working_dir):
'''build an image if called for based on having a recipe and context.
Otherwise, pull a container uri to the instance workspace.
'''
sif_binary = self.get_image()
# If the final image already exists, don't continue
if os.path.exists(sif_binary):
return
# Case 1: Given an image
if self.image is not None:
if not os.path.exists(self.image):
# Can we pull it?
if re.search('(docker|library|shub|http?s)[://]', self.image):
bot.info('Pulling %s' % self.image)
self.client.pull(self.image, name=sif_binary)
else:
bot.exit('%s is an invalid unique resource identifier.' %
self.image)
# Case 2: Given a recipe
elif self.recipe is not None:
# Change directory to the context
context = os.path.abspath(self.context)
os.chdir(context)
# The recipe is expected to exist in the context folder
if not os.path.exists(self.recipe):
bot.exit('%s not found for build' % self.recipe)
# This will likely require sudo, unless --remote or --fakeroot in options
try:
options = self.get_build_options()
# If remote or fakeroot included, don't need sudo
sudo = not ("--fakeroot" in options or "--remote" in options)
bot.info('Building %s' % self.name)
_, stream = self.client.build(image=sif_binary,
recipe=self.recipe,
options=options,
sudo=sudo,
stream=True)
for line in stream:
print(line)
except:
build = "sudo singularity build %s %s" % (
os.path.basename(sif_binary), self.recipe)
bot.warning("Issue building container, try: %s" % build)
# Change back to provided working directory
os.chdir(working_dir)
else:
bot.exit("neither image and build defined for %s" % self.name)
def _create(
self,
names,
command="create",
writable_tmpfs=True,
bridge="10.22.0.0/16",
no_resolv=False,
):
"""create one or more instances. "Command" determines the sub function
to call for the instance, which should be "create" or "up".
If the user provide a list of names, use them, otherwise default
to all instances.
Parameters
==========
names: the instance names to create
command: one of "create" or "up"
writable_tmpfs: if the instances should be given writable to tmp
bridge: the bridge ip address to use for networking, and generating
addresses for the individual containers.
see /usr/local/etc/singularity/network/00_bridge.conflist
no_resolv: if True, don't create and bind a resolv.conf with Google
nameservers.
"""
# If no names provided, we create all
names = names or self.get_instance_names()
# Keep track of created instances to determine if we have circular dependency structure
created = []
circular_dep = False
# Generate ip addresses for each
lookup = self.get_ip_lookup(names, bridge)
if self.sudo and not no_resolv:
# Generate shared hosts file
hosts_file = self.create_hosts(lookup)
for instance in self.iter_instances(names):
depends_on = instance.params.get("depends_on", [])
for dep in depends_on:
if dep not in created:
circular_dep = True
# Generate a resolv.conf to bind to the container
if self.sudo and not no_resolv:
resolv = self.generate_resolv_conf()
instance.volumes.append("%s:/etc/resolv.conf" % resolv)
# Create a hosts file for the instance based, add as volume
instance.volumes.append("%s:/etc/hosts" % hosts_file)
# If we get here, execute command and add to list
create_func = getattr(instance, command)
create_func(
working_dir=self.working_dir,
writable_tmpfs=writable_tmpfs,
ip_address=lookup[instance.name],
)
created.append(instance.name)
# Run post create commands
instance.run_post()
if circular_dep:
bot.exit(
"Unable to create all instances, possible circular dependency."
)
def _create(self,
names,
command="create",
writable_tmpfs=True,
bridge="10.22.0.0/16",
no_resolv=False):
'''create one or more instances. "Command" determines the sub function
to call for the instance, which should be "create" or "up".
If the user provide a list of names, use them, otherwise default
to all instances.
Parameters
==========
names: the instance names to create
command: one of "create" or "up"
writable_tmpfs: if the instances should be given writable to tmp
bridge: the bridge ip address to use for networking, and generating
addresses for the individual containers.
see /usr/local/etc/singularity/network/00_bridge.conflist
no_resolv: if True, don't create and bind a resolv.conf with Google
nameservers.
'''
# If no names provided, we create all
names = names or self.get_instance_names()
# Keep a count to determine if we have circular dependency structure
created = []
count = 0
# Generate ip addresses for each
lookup = self.get_ip_lookup(names, bridge)
# Generate shared hosts file
hosts_file = self.create_hosts(lookup)
# First create those with no dependencies
while names:
for instance in self.iter_instances(names):
# Flag to indicated create
do_create = True
# Ensure created, skip over if not
depends_on = instance.params.get('depends_on', [])
for depends_on in depends_on:
if depends_on not in created:
count += 1
do_create = False
if do_create:
# Generate a resolv.conf to bind to the container
if not no_resolv:
resolv = self.generate_resolv_conf()
instance.volumes.append('%s:/etc/resolv.conf' % resolv)
# Create a hosts file for the instance based, add as volume
instance.volumes.append('%s:/etc/hosts' % hosts_file)
# If we get here, execute command and add to list
create_func = getattr(instance, command)
create_func(working_dir=self.working_dir,
writable_tmpfs=writable_tmpfs,
ip_address=lookup[instance.name])
created.append(instance.name)
names.remove(instance.name)
# Run post create commands
instance.run_post()
# Possibly circular dependencies
if count >= 100:
bot.exit(
'Unable to create all instances, possible circular dependency.'
)
