def _multistage(self, fromHeader):
"""Given a from header, determine if we have a multistage build, and
update the recipe parser active in case that we do. If we are dealing
with the first layer and it's named, we also update the default
name "spython-base" to be what the recipe intended.
Parameters
==========
fromHeader: the fromHeader parsed from self.from, possibly with AS
"""
# Derive if there is a named layer
match = re.search("AS (?P<layer>.+)", fromHeader, flags=re.I)
if match:
layer = match.groups("layer")[0].strip()
# If it's the first layer named incorrectly, we need to rename
if len(self.recipe) == 1 and list(
self.recipe)[0] == "spython-base":
self.recipe[layer] = deepcopy(self.recipe[self.active_layer])
del self.recipe[self.active_layer]
else:
self.active_layer_num += 1
self.recipe[layer] = Recipe(self.filename,
self.active_layer_num)
self.active_layer = layer
bot.debug("Active layer #%s updated to %s" %
(self.active_layer_num, self.active_layer))
def parse(self):
"""parse is the base function for parsing the recipe, and extracting
elements into the correct data structures. Everything is parsed into
lists or dictionaries that can be assembled again on demand.
Singularity: we parse files/labels first, then install.
cd first in a line is parsed as WORKDIR
"""
# If the recipe isn't loaded, load it
if not hasattr(self, "config"):
self.load_recipe()
# Parse each section
for section, lines in self.config.items():
bot.debug(section)
# Get the correct parsing function
parser = self._get_mapping(section)
# Parse it, if appropriate
if parser:
parser(lines)
return self.recipe
def _add_section(self, line, section=None):
'''parse a line for a section, and return the parsed section (if not
None)
Parameters
==========
line: the line to parse
section: the current (or previous) section
Resulting data structure is:
config['post'] (in lowercase)
'''
# Remove any comments
line = line.split('#', 1)[0].strip()
# Is there a section name?
parts = line.split(' ')
if len(parts) > 1:
name = ' '.join(parts[1:])
section = re.sub('[%]|(\s+)', '', parts[0]).lower()
if section not in self.config:
self.config[section] = []
bot.debug("Adding section %s" % section)
return section
def _add_section(self, line, section=None):
"""parse a line for a section, and return the parsed section (if not
None)
Parameters
==========
line: the line to parse
section: the current (or previous) section
Resulting data structure is:
config['post'] (in lowercase)
"""
# Remove any comments
line = line.split("#", 1)[0].strip()
# Is there a section name?
parts = line.split(" ")
section = re.sub(r"[%]|(\s+)", "", parts[0]).lower()
if section not in self.config:
self.config[section] = []
bot.debug("Adding section %s" % section)
return section
def _arg(self, line):
"""singularity doesn't have support for ARG, so instead will issue
a warning to the console for the user to export the variable
with SINGULARITY prefixed at build.
Parameters
==========
line: the line from the recipe file to parse for ARG
"""
line = self._setup("ARG", line)
# Args are treated like envars, so we add them to install
environ = self.parse_env([x for x in line if "=" in x])
self.recipe[self.active_layer].install += environ
# Try to extract arguments from the line
for arg in line:
# An undefined arg cannot be used
if "=" not in arg:
bot.warning(
"ARG is not supported for Singularity, and must be defined with "
"a default to be parsed. Skipping %s" % arg)
continue
arg, value = arg.split("=", 1)
arg = arg.strip()
value = value.strip()
bot.debug("Updating ARG %s to %s" % (arg, value))
self.args[arg] = value
def _from(self, line):
"""get the FROM container image name from a FROM line!
Parameters
==========
line: the line from the recipe file to parse for FROM
"""
self.recipe[self.active_layer].fromHeader = line
bot.debug("FROM %s" % self.recipe[self.active_layer].fromHeader)
def _from(self, line):
''' get the FROM container image name from a FROM line!
Parameters
==========
line: the line from the recipe file to parse for FROM
'''
self.fromHeader = line
bot.debug('FROM %s' % self.fromHeader)
def _setup(self, action, line):
''' replace the command name from the group, alert the user of content,
and clean up empty spaces
'''
bot.debug('[in] %s' % line)
# Replace ACTION at beginning
line = re.sub('^%s' % action, '', line)
# Split into components
return [x for x in self._split_line(line) if x not in ['', None]]
def load_recipe(self):
"""load_recipe will return a loaded in singularity recipe. The idea
is that these sections can then be parsed into a Dockerfile,
or printed back into their original form.
Returns
=======
config: a parsed recipe Singularity recipe
"""
# Comments between sections, add to top of file
lines = self.lines[:]
fromHeader = None
stage = None
comments = []
while lines:
# Clean up white trailing/leading space
line = lines.pop(0)
stripped = line.strip()
# Bootstrap Line
if re.search("bootstrap", line, re.IGNORECASE):
self._check_bootstrap(stripped)
section = None
# From Line
elif re.search("from:", stripped, re.IGNORECASE):
fromHeader = stripped
if stage is None:
self._load_from(fromHeader)
# Identify stage
elif re.search("stage:", stripped, re.IGNORECASE):
stage = re.sub("stage:", "", stripped.lower()).strip()
self._multistage("as %s" % stage)
self._load_from(fromHeader)
# Comment
elif stripped.startswith("#") and stripped not in comments:
comments.append(stripped)
# Section
elif stripped.startswith("%"):
section, layer = self._get_section(stripped)
bot.debug("Found section %s" % section)
# If we have a section, and are adding it
elif section is not None:
lines = [line] + lines
self._load_section(lines=lines, section=section, layer=layer)
self._comments(comments)
def version(self):
'''return the version of singularity
'''
if not check_install():
bot.warning("Singularity version not found, so it's likely not installed.")
else:
cmd = ['singularity','--version']
version = self._run_command(cmd).strip('\n')
bot.debug("Singularity %s being used." % version)
return version
def _from(self, line):
''' get the FROM container image name from a FROM line!
Parameters
==========
line: the line from the recipe file to parse for FROM
'''
self.fromHeader = self._setup('FROM', line)
if "scratch" in self.fromHeader:
bot.warning('scratch is no longer available on Docker Hub.')
bot.debug('FROM %s' % self.fromHeader)
def load_recipe(self):
'''load will return a loaded in singularity recipe. The idea
is that these sections can then be parsed into a Dockerfile,
or printed back into their original form.
Returns
=======
config: a parsed recipe Singularity recipe
'''
# Comments between sections, add to top of file
lines = self.lines.copy()
comments = []
# Start with a fresh config!
self.config = dict()
section = None
name = None
while len(lines) > 0:
# Clean up white trailing/leading space
line = lines.pop(0)
stripped = line.strip()
# Bootstrap Line
if re.search('(b|B)(o|O){2}(t|T)(s|S)(t|T)(r|R)(a|A)(p|P)', line):
self._load_bootstrap(stripped)
# From Line
if re.search('(f|F)(r|R)(O|o)(m|M)', stripped):
self._load_from(stripped)
# Comment
if stripped.startswith("#"):
comments.append(stripped)
continue
# Section
elif stripped.startswith('%'):
section = self._add_section(stripped)
bot.debug("Adding section title %s" %section)
# If we have a section, and are adding it
elif section is not None:
lines = [line] + lines
self._load_section(lines=lines,
section=section)
self.config['comments'] = comments
def _setup(self, action, line):
"""replace the command name from the group, alert the user of content,
and clean up empty spaces
"""
bot.debug("[in] %s" % line)
# Replace ACTION at beginning
line = re.sub("^%s" % action, "", line)
# Handle continuation lines without ACTION by padding with leading space
line = " " + line
# Split into components
return [x for x in self._split_line(line) if x not in ["", None]]
def _from(self, line):
""" get the FROM container image name from a FROM line!
Parameters
==========
line: the line from the recipe file to parse for FROM
recipe: the recipe object to populate.
"""
fromHeader = self._setup("FROM", line)
# Singularity does not support AS level
self.recipe.fromHeader = re.sub("AS .+", "", fromHeader[0], flags=re.I)
if "scratch" in self.recipe.fromHeader:
bot.warning("scratch is no longer available on Docker Hub.")
bot.debug("FROM %s" % self.recipe.fromHeader)
def _from(self, line):
''' get the FROM container image name from a FROM line!
Parameters
==========
line: the line from the recipe file to parse for FROM
'''
fromHeader = self._setup('FROM', line)
# Singularity does not support AS level
self.fromHeader = re.sub("AS .+", "", fromHeader[0], flags=re.I)
if "scratch" in self.fromHeader:
bot.warning('scratch is no longer available on Docker Hub.')
bot.debug('FROM %s' % self.fromHeader)
def generate_bind_list(self, bindlist=None):
"""generate bind string will take a single string or list of binds, and
return a list that can be added to an exec or run command. For example,
the following map as follows:
['/host:/container', '/both'] --> ["--bind", "/host:/container","--bind","/both" ]
['/both'] --> ["--bind", "/both"]
'/host:container' --> ["--bind", "/host:container"]
None --> []
An empty bind or otherwise value of None should return an empty list.
The binds are also checked on the host.
Parameters
==========
bindlist: a string or list of bind mounts
"""
binds = []
# Case 1: No binds provided
if not bindlist:
return binds
# Case 2: provides a long string or non list, and must be split
if not isinstance(bindlist, list):
bindlist = bindlist.split(" ")
for bind in bindlist:
# Still cannot be None
if bind:
bot.debug("Adding bind %s" % bind)
binds += ["--bind", bind]
# Check that exists on host
host = bind.split(":")[0]
if not os.path.exists(host):
bot.error("%s does not exist on host." % bind)
sys.exit(1)
return binds
def set_verbosity(args):
'''determine the message level in the environment to set based on args.
'''
level = "INFO"
if args.debug:
level = "DEBUG"
elif args.quiet:
level = "QUIET"
os.environ['MESSAGELEVEL'] = level
os.putenv('MESSAGELEVEL', level)
os.environ['SINGULARITY_MESSAGELEVEL'] = level
os.putenv('SINGULARITY_MESSAGELEVEL', level)
# Import logger to set
from spython.logger import bot
bot.debug('Logging level %s' %level)
import spython
bot.debug("Singularity Python Version: %s" % spython.__version__)
def _create_section(self, attribute, name=None, stage=None):
"""create a section based on key, value recipe pairs,
This is used for files or label
Parameters
==========
attribute: the name of the data section, either labels or files
name: the name to write to the recipe file (e.g., %name).
if not defined, the attribute name is used.
"""
# Default section name is the same as attribute
if name is None:
name = attribute
# Put a space between sections
section = ["\n"]
# Only continue if we have the section and it's not empty
try:
section = getattr(self.recipe[self.stage], attribute)
except AttributeError:
bot.debug("Recipe does not have section for %s" % attribute)
return section
# if the section is empty, don't print it
if not section:
return section
# Files
if attribute in ["files", "labels"]:
return create_keyval_section(section, name, stage)
# An environment section needs exports
if attribute in ["environ"]:
return create_env_section(section, name)
# Post, Setup
return finish_section(section, name)
def _from(self, line):
"""get the FROM container image name from a FROM line. If we have
already seen a FROM statement, this is indicative of adding
another image (multistage build).
Parameters
==========
line: the line from the recipe file to parse for FROM
recipe: the recipe object to populate.
"""
fromHeader = self._setup("FROM", line)
# Do we have a multistge build to update the active layer?
self._multistage(fromHeader[0])
# Now extract the from header, make args replacements
self.recipe[self.active_layer].fromHeader = self._replace_from_dict(
re.sub("AS .+", "", fromHeader[0], flags=re.I), self.args)
if "scratch" in self.recipe[self.active_layer].fromHeader:
bot.warning("scratch is no longer available on Docker Hub.")
bot.debug("FROM %s" % self.recipe[self.active_layer].fromHeader)