def _copy(self, lines):
"""parse_add will copy multiple files from one location to another.
This likely will need tweaking, as the files might need to be
mounted from some location before adding to the image.
The add command is done for an entire directory. It is also
possible to have more than one file copied to a destination:
https://docs.docker.com/engine/reference/builder/#copy
e.g.: <src> <src> <dest>/
"""
lines = self._setup("COPY", lines)
for line in lines:
# Take into account multistage builds
layer = None
if line.startswith("--from"):
layer = line.strip("--from").split(" ")[0].lstrip("=")
if layer not in self.recipe:
bot.warning(
"COPY requested from layer %s, but layer not previously defined."
% layer)
continue
# Remove the --from from the line
line = " ".join([l for l in line.split(" ")[1:] if l])
values = line.split(" ")
topath = values.pop()
for frompath in values:
self._add_files(frompath, topath, layer)
def _add_files(self, source, dest):
'''add files is the underlying function called to add files to the
list, whether originally called from the functions to parse archives,
or https. We make sure that any local references are changed to
actual file locations before adding to the files list.
Parameters
==========
source: the source
dest: the destiation
'''
# Create data structure to iterate over
paths = {'source': source, 'dest': dest}
for pathtype, path in paths.items():
if path == ".":
paths[pathtype] = os.getcwd()
# Warning if doesn't exist
if not os.path.exists(path):
bot.warning("%s doesn't exist, ensure exists for build" % path)
# The pair is added to the files as a list
self.files.append([paths['source'], paths['dest']])
def _logs(self, print_logs=False, ext="out"):
"""A shared function to print log files. The only differing element is
the extension (err or out)
"""
from spython.utils import check_install
check_install()
# Formulate the path of the logs
hostname = platform.node()
logpath = os.path.join(
get_userhome(),
".singularity",
"instances",
"logs",
hostname,
get_username(),
"%s.%s" % (self.name, ext),
)
if os.path.exists(logpath):
with open(logpath, "r") as filey:
logs = filey.read()
if print_logs is True:
print(logs)
else:
bot.warning("No log files have been produced.")
return logs
def _add_files(self, source, dest, layer=None):
"""add files is the underlying function called to add files to the
list, whether originally called from the functions to parse archives,
or https. We make sure that any local references are changed to
actual file locations before adding to the files list.
Parameters
==========
source: the source
dest: the destiation
"""
# Warn the user Singularity doesn't support expansion
if "*" in source:
bot.warning(
"Singularity doesn't support expansion, * found in %s" %
source)
# Warning if file/folder (src) doesn't exist
if not os.path.exists(source) and layer is None:
bot.warning("%s doesn't exist, ensure exists for build" % source)
# The pair is added to the files as a list
if not layer:
self.recipe[self.active_layer].files.append([source, dest])
# Unless the file is to be copied from a particular layer
else:
if layer not in self.recipe[self.active_layer].layer_files:
self.recipe[self.active_layer].layer_files[layer] = []
self.recipe[self.active_layer].layer_files[layer].append(
[source, dest])
def _arg(self, line):
"""singularity doesn't have support for ARG, so instead will issue
a warning to the console for the user to export the variable
with SINGULARITY prefixed at build.
Parameters
==========
line: the line from the recipe file to parse for ARG
"""
line = self._setup("ARG", line)
# Args are treated like envars, so we add them to install
environ = self.parse_env([x for x in line if "=" in x])
self.recipe[self.active_layer].install += environ
# Try to extract arguments from the line
for arg in line:
# An undefined arg cannot be used
if "=" not in arg:
bot.warning(
"ARG is not supported for Singularity, and must be defined with "
"a default to be parsed. Skipping %s" % arg)
continue
arg, value = arg.split("=", 1)
arg = arg.strip()
value = value.strip()
bot.debug("Updating ARG %s to %s" % (arg, value))
self.args[arg] = value
def export(
self,
image_path,
pipe=False,
output_file=None,
command=None,
sudo=False,
singularity_options=None,
):
"""export will export an image, sudo must be used. If we have Singularity
versions after 3, export is replaced with building into a sandbox.
Parameters
==========
image_path: full path to image
pipe: export to pipe and not file (default, False)
singularity_options: a list of options to provide to the singularity client
output_file: if pipe=False, export tar to this file. If not specified,
will generate temporary directory.
"""
from spython.utils import check_install
check_install()
if "version 3" in self.version() or "2.6" in self.version():
# If export is deprecated, we run a build
bot.warning(
"Export is not supported for Singularity 3.x. Building to sandbox instead."
)
if output_file is None:
basename, _ = os.path.splitext(image_path)
output_file = self._get_filename(basename, "sandbox", pwd=False)
return self.build(
recipe=image_path,
image=output_file,
sandbox=True,
force=True,
sudo=sudo,
singularity_options=singularity_options,
)
# If not version 3, run deprecated command
elif "2.5" in self.version():
return self._export(
image_path=image_path,
pipe=pipe,
output_file=output_file,
command=command,
singularity_options=singularity_options,
)
bot.warning("Unsupported version of Singularity, %s" % self.version())
def version(self):
'''return the version of singularity
'''
if not check_install():
bot.warning("Singularity version not found, so it's likely not installed.")
else:
cmd = ['singularity','--version']
version = self._run_command(cmd).strip('\n')
bot.debug("Singularity %s being used." % version)
return version
def _from(self, line):
''' get the FROM container image name from a FROM line!
Parameters
==========
line: the line from the recipe file to parse for FROM
'''
self.fromHeader = self._setup('FROM', line)
if "scratch" in self.fromHeader:
bot.warning('scratch is no longer available on Docker Hub.')
bot.debug('FROM %s' % self.fromHeader)
def _arg(self, line):
"""singularity doesn't have support for ARG, so instead will issue
a warning to the console for the user to export the variable
with SINGULARITY prefixed at build.
Parameters
==========
line: the line from the recipe file to parse for ARG
"""
line = self._setup("ARG", line)
bot.warning("ARG is not supported for Singularity! To get %s" % line[0])
bot.warning("in the container, on host export SINGULARITY_%s" % line[0])
def _from(self, line):
''' get the FROM container image name from a FROM line!
Parameters
==========
line: the line from the recipe file to parse for FROM
'''
fromHeader = self._setup('FROM', line)
# Singularity does not support AS level
self.fromHeader = re.sub("AS .+", "", fromHeader[0], flags=re.I)
if "scratch" in self.fromHeader:
bot.warning('scratch is no longer available on Docker Hub.')
bot.debug('FROM %s' % self.fromHeader)
def _from(self, line):
""" get the FROM container image name from a FROM line!
Parameters
==========
line: the line from the recipe file to parse for FROM
recipe: the recipe object to populate.
"""
fromHeader = self._setup("FROM", line)
# Singularity does not support AS level
self.recipe.fromHeader = re.sub("AS .+", "", fromHeader[0], flags=re.I)
if "scratch" in self.recipe.fromHeader:
bot.warning("scratch is no longer available on Docker Hub.")
bot.debug("FROM %s" % self.recipe.fromHeader)
def export(self,
image_path,
pipe=False,
output_file=None,
command=None,
sudo=False):
'''export will export an image, sudo must be used. If we have Singularity
versions after 3, export is replaced with building into a sandbox.
Parameters
==========
image_path: full path to image
pipe: export to pipe and not file (default, False)
output_file: if pipe=False, export tar to this file. If not specified,
will generate temporary directory.
'''
from spython.utils import check_install
check_install()
if 'version 3' in self.version() or '2.6' in self.version():
# If export is deprecated, we run a build
bot.warning(
'Export is not supported for Singularity 3.x. Building to sandbox instead.'
)
if output_file is None:
basename, _ = os.path.splitext(image_path)
output_file = self._get_filename(basename, 'sandbox', pwd=False)
return self.build(recipe=image_path,
image=output_file,
sandbox=True,
force=True,
sudo=sudo)
# If not version 3, run deprecated command
elif '2.5' in self.version():
return self._export(image_path=image_path,
pipe=pipe,
output_file=output_file,
command=command)
bot.warning('Unsupported version of Singularity, %s' % self.version())
def get_hash(self, image=None):
"""return an md5 hash of the file based on a criteria level. This
is intended to give the file a reasonable version. This only is
useful for actual image files.
Parameters
==========
image: the image path to get hash for (first priority). Second
priority is image path saved with image object, if exists.
"""
hasher = hashlib.md5()
image = image or self.image
if os.path.exists(image):
with open(image, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hasher.update(chunk)
return hasher.hexdigest()
bot.warning("%s does not exist." % image)
def _load_section(self, lines, section, layer=None):
"""read in a section to a list, and stop when we hit the next section"""
members = []
while True:
if not lines:
break
next_line = lines[0]
# We have a start of another bootstrap
if re.search("bootstrap:", next_line, re.IGNORECASE):
break
# The end of a section
if next_line.strip().startswith("%"):
break
# Still in current section!
else:
new_member = lines.pop(0).strip()
if new_member not in ["", None]:
members.append(new_member)
# Add the list to the config
if members and section is not None:
# Get the correct parsing function
parser = self._get_mapping(section)
# Parse it, if appropriate
if not parser:
bot.warning("%s is an unrecognized section, skipping." %
section)
else:
if section == "files":
parser(members, layer)
else:
parser(members)
def _from(self, line):
"""get the FROM container image name from a FROM line. If we have
already seen a FROM statement, this is indicative of adding
another image (multistage build).
Parameters
==========
line: the line from the recipe file to parse for FROM
recipe: the recipe object to populate.
"""
fromHeader = self._setup("FROM", line)
# Do we have a multistge build to update the active layer?
self._multistage(fromHeader[0])
# Now extract the from header, make args replacements
self.recipe[self.active_layer].fromHeader = self._replace_from_dict(
re.sub("AS .+", "", fromHeader[0], flags=re.I), self.args)
if "scratch" in self.recipe[self.active_layer].fromHeader:
bot.warning("scratch is no longer available on Docker Hub.")
bot.debug("FROM %s" % self.recipe[self.active_layer].fromHeader)
def _run(self, lines):
"""_parse the runscript to be the Docker CMD. If we have one line,
call it directly. If not, write the entrypoint into a script.
Parameters
==========
lines: the line from the recipe file to parse for CMD
"""
lines = [x for x in lines if x not in ["", None]]
# Default runscript is first index
runscript = lines[0]
# Multiple line runscript needs multiple lines written to script
if len(lines) > 1:
bot.warning("More than one line detected for runscript!")
bot.warning("These will be echoed into a single script to call.")
self._write_script("/entrypoint.sh", lines)
runscript = "/bin/bash /entrypoint.sh"
self.recipe[self.active_layer].cmd = runscript
def _add_files(self, source, dest):
'''add files is the underlying function called to add files to the
list, whether originally called from the functions to parse archives,
or https. We make sure that any local references are changed to
actual file locations before adding to the files list.
Parameters
==========
source: the source
dest: the destiation
'''
# Warn the user Singularity doesn't support expansion
if '*' in source:
bot.warning(
"Singularity doesn't support expansion, * found in %s" %
source)
# Warning if file/folder (src) doesn't exist
if not os.path.exists(source):
bot.warning("%s doesn't exist, ensure exists for build" % source)
# The pair is added to the files as a list
self.recipe.files.append([source, dest])
def _setup(self, lines):
"""setup required adding content from the host to the rootfs,
so we try to capture with with ADD.
"""
bot.warning("SETUP is error prone, please check output.")
for line in lines:
# For all lines, replace rootfs with actual root /
line = re.sub("[$]{?SINGULARITY_ROOTFS}?", "",
"$SINGULARITY_ROOTFS")
# If we have nothing left, don't continue
if line in ["", None]:
continue
# If the line starts with copy or move, assume is file from host
if re.search("(^cp|^mv)", line):
line = re.sub("(^cp|^mv)", "", line)
self.files.append(line)
# If it's a general command, add to install routine
else:
self.install.append(line)
