def __init__(self, **kwargs):
self.platform = Platform_Meta('Singularity', 'Ⓢ')
image = kwargs.get("image")
tag = kwargs.get("tag")
pwd = os.getcwd()
if kwargs.get("working_dir") is not None:
pwd = kwargs["working_dir"]
os.chdir(pwd)
print(f"Loading {self.platform.symbol} {self.platform.name}")
if image and isinstance(image, str) and os.path.exists(image):
self.image = image
elif tag and isinstance(tag, str): # pragma: no cover
self.image = Client.pull(f"docker://{image}:{tag}",
pull_folder=pwd)
else: # pragma: no cover
try:
self.image = Client.pull("shub://FCP-INDI/C-PAC",
pull_folder=pwd)
except Exception:
try:
self.image = Client.pull(f"docker://fcpindi/c-pac:latest",
pull_folder=pwd)
except Exception:
raise OSError("Could not connect to Singularity")
self.volumes = {}
self.options = list(chain.from_iterable(
kwargs["container_options"])) if bool(
kwargs.get("container_options")) else []
self._set_bindings(**kwargs)
def singularity_pull(self, image):
"""Pulls an docker or singularity images from hub.
"""
log.info('{}[{}] singularity pull {}'.format(
self.msg_prefix, self.action['name'], image)
)
if not self.dry_run:
sclient.pull(image, name=self.image_name)
def download_images(image_dir, image_url):
image_name = 'su2_containers_fork_dev.sif'
try:
os.mkdir(image_dir)
except:
pass
client.pull(image=image_url, name=image_name, pull_folder=image_dir)
return image_name
def singularity_pull(self, image):
"""Pulls an docker or singularity images from hub.
"""
if not self.skip_pull:
log.info('{}[{}] singularity pull {}'.format(
self.msg_prefix, self.action['name'], image))
if not self.dry_run:
sclient.pull(image, name=self.image_name)
else:
if not self.singularity_exists():
log.fail('The required singularity image {} was not found '
'locally.'.format(self.image_name))
def run_singularity_container_executable(container_image, executable_path,
args_dict):
"""Launch an executable within a Singularity container.
Args:
container_image: A string containing the name of the container
to pull.
executable_path: A string containing the path of the executable
to execute within the container.
args_dict: A dictionary containing arguments and corresponding
values.
"""
# Import Singularity library
from spython.main import Client as client
# Pull the specified container. This pull in the latest version of
# the container (with the specified tag if provided).
singularity_image = client.pull(container_image)
# Run the executable with the arguments
# TODO how do we get logs?
client.run(
singularity_image,
[executable_path, json.dumps(args_dict)],
)
# TODO give a real return value
return "FINISHED"
def get_nvidia_devices(use_docker=True):
"""
Returns a Dict[index, UUID] of all NVIDIA devices available to docker
Arguments:
use_docker: whether or not to use a docker container to run nvidia-smi. if not, use singularity
Raises docker.errors.ContainerError if GPUs are unreachable,
docker.errors.ImageNotFound if the CUDA image cannot be pulled
docker.errors.APIError if another server error occurs
"""
cuda_image = 'nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04'
nvidia_command = 'nvidia-smi --query-gpu=index,uuid --format=csv,noheader'
if use_docker:
client.images.pull(cuda_image)
output = client.containers.run(
cuda_image,
nvidia_command,
runtime=NVIDIA_RUNTIME,
detach=False,
stdout=True,
remove=True,
)
gpus = output.decode()
else:
# use the singularity runtime to run nvidia-smi
img = Client.pull('docker://' + cuda_image, pull_folder='/tmp')
output = Client.execute(img, nvidia_command, options=['--nv'])
if output['return_code'] != 0:
raise SingularityError
gpus = output['message']
# Get newline delimited gpu-index, gpu-uuid list
logger.info("GPUs: " + str(gpus.split('\n')[:-1]))
return {gpu.split(',')[0].strip(): gpu.split(',')[1].strip() for gpu in gpus.split('\n')[:-1]}
def action_pull():
'''the fetch view to perform the pull, and return a response
'''
# Ensure uri appears once
container = request.form.get('uri') # ubuntu:latest
uri = request.form.get('endpoint') # docker://
container = '%s%s' % (uri, container.replace(uri, ''))
app.logger.info("PULL for %s" % container)
# If nvidia is used, use sregistry client
if 'nvidia' in uri:
nvidia = app.sregistry._get_setting('SREGISTRY_NVIDIA_TOKEN')
# Make sure we use the right client
os.environ['SREGISTRY_CLIENT'] = uri.replace('://', '')
os.environ.putenv('SREGISTRY_CLIENT', uri.replace('://', ''))
from sregistry.main import get_client
client = get_client(image=container)
app.logger.info("Using client %s" % client.client_name)
try:
puller = client.pull(container, force=True)
except:
puller = '''ERROR: manifest unknown, or pull error.
Use docker-compose logs web to see issue!'''
else:
# We will stream the response back!
image, puller = Client.pull(container, stream=True, pull_folder='/tmp')
puller = itertools.chain(puller, [image])
return Response(puller, mimetype='text/plain')
def __init__(self, mode, container_image, volumes, extra_kwargs):
assert mode in ('docker', 'singularity')
self.mode = mode
if mode == 'docker':
import docker
client = docker.from_env()
if extra_kwargs.get('requires_gpu', False):
extra_kwargs.pop('requires_gpu')
extra_kwargs["device_requests"] = [
docker.types.DeviceRequest(count=-1,
capabilities=[['gpu']])
]
# check if the image is already present locally
repo_tags = []
for image in client.images.list():
repo_tags.extend(image.attrs['RepoTags'])
if container_image not in repo_tags:
client.images.pull(container_image)
self.docker_container = client.containers.create(container_image,
tty=True,
volumes=volumes,
**extra_kwargs)
elif mode == 'singularity':
from spython.main import Client
# load local image file if it exists, otherwise search dockerhub
if Path(container_image).exists():
self.singularity_image = container_image
else:
self.singularity_image = Client.pull(
f'docker://{container_image}')
if not Path(self.singularity_image).exists():
raise FileNotFoundError(
f'Unable to locate container image {container_image}')
# bin options
singularity_bind = ','.join([
f'{volume_src}:{volume["bind"]}'
for volume_src, volume in volumes.items()
])
options = ['--bind', singularity_bind]
# gpu options
if extra_kwargs.get('requires_gpu', False):
# only nvidia at the moment
options += ['--nv']
self.client_instance = Client.instance(self.singularity_image,
start=False,
options=options)
def test_pull_and_run(tmp_path):
image = Client.pull("shub://vsoch/singularity-images",
pull_folder=str(tmp_path))
print(image)
assert os.path.exists(image)
ext = 'sif' if Client.version_info().major >= 3 else 'simg'
assert image == str(tmp_path / ('singularity-images.' + ext))
result = Client.run(image)
print(result)
assert 'You say please, but all I see is pizza..' in result
def pull_image(self, model, progress_stream=sys.stderr):
if len(set(model.platforms) & {"shub", "library"}) == 0:
if "singularity" in model.platforms:
# It's a local image. Just check that it exists, and raise if
# not.
if not self.image_exists(model):
raise ValueError("Could not find local Singularity image at %s" % (model.reference,))
else:
raise ValueError("Only know how to pull from shub:// and library://"
" . This Singularity model does not come from "
"either repository.")
return Client.pull(image="%s://%s" % (model.repository, model.reference))
def _download(self, image_spec: str):
logger.debug('Downloading image %s', image_spec)
try:
# stream=True for singularity doesnt return progress or anything really - for now no progress
self._downloading[image_spec]['message'] = "Starting download"
img = Client.pull(image_spec, pull_folder=self.image_folder)
logger.debug('Download for image %s complete to %s', image_spec,
img)
self._downloading[image_spec]['success'] = True
self._downloading[image_spec]['message'] = "Downloaded image"
except Exception as ex:
logger.debug('Download for Singularity image %s failed: %s',
image_spec, ex)
self._downloading[image_spec]['success'] = False
self._downloading[image_spec][
'message'] = "Can't download image: {}".format(ex)
def run_singularity_container_executable(uuid, container_image,
executable_path, logs_path,
args_dict):
"""Launch an executable within a Singularity container.
Args:
uuid: A string containing the uuid of the job being run.
container_image: A string containing the name of the container
to pull.
executable_path: A string containing the path of the executable
to execute within the container.
logs_path: A string (or None) containing the path of the
directory containing the relevant logs within the container.
args_dict: A dictionary containing arguments and corresponding
values.
"""
# Import Singularity library
from spython.main import Client as client
# Pull the specified container. This pull in the latest version of
# the container (with the specified tag if provided).
singularity_image = client.pull(
image=container_image,
pull_folder=os.environ['WORKER_SINGULARITY_IMAGES_DIRECTORY'],
)
# Run the executable with the arguments
if logs_path is None:
bind_option = None
else:
# Create the host path. This is required by the Singularity
# library (though not the Docker library)
host_path = os.path.join(os.environ['WORKER_LOGS_DIRECTORY'], uuid)
os.makedirs(host_path, exist_ok=True)
# Build the bind option to pass on to Singularity
bind_option = host_path.rstrip('/') + ":" + logs_path.rstrip('/')
client.execute(
image=singularity_image,
command=[executable_path, json.dumps(args_dict)],
bind=bind_option,
)
# TODO give a real return value
return "FINISHED"
def _pull(self,
file_name,
names,
save=True,
force=False,
uri="docker://",
**kwargs):
"""pull an image from a docker hub. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers, metadata folder, and custom metadata to it
- converts to a squashfs image with build
the docker manifests are stored with registry metadata.
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
"""
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(names)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.exit("Image exists! Remove first, or use --force to overwrite")
digest = names["version"] or names["tag"]
# Build from sandbox, prefix with sandbox
sandbox = get_tmpdir(prefix="sregistry-sandbox")
# First effort, get image via Sregistry
layers = self._download_layers(names["url"], digest)
# This is the url where the manifests were obtained
url = self._get_manifest_selfLink(names["url"], digest)
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info("Exploding %s" % layer)
result = extract_tar(layer, sandbox, handle_whiteout=True)
if result["return_code"] != 0:
bot.exit(result["message"])
sudo = kwargs.get("sudo", False)
# Build from a sandbox (recipe) into the image_file (squashfs)
image_file = Singularity.build(image=file_name, recipe=sandbox, sudo=sudo)
# Fall back to using Singularity
if image_file is None:
bot.info("Downloading with native Singularity, please wait...")
image = file_name.replace("docker://", uri)
image_file = Singularity.pull(image, pull_folder=sandbox)
# Save to local storage
if save is True:
# Did we get the manifests?
manifests = {}
if hasattr(self, "manifests"):
manifests = self.manifests
container = self.add(image_path=image_file,
image_uri=names["uri"],
metadata=manifests,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug("Retrieved image file %s" % image_file)
bot.custom(prefix="Success!", message=image_file)
# Clean up sandbox
shutil.rmtree(sandbox)
return image_file
def pull_image(self, output_name=None, force=False):
self.image_installed = False
# the name must be valid. we use _ to separate name and version for the
# singularity filemane but the image have a tag (version) and name
# separated by a :
# Our registry uses underscore. The output filename should use _ as well
# However, for users and image stored on sylabs or singularity hub, we
# use the character :
# let us first retrieve the image and save it in a temporary file.
# we keep the original name by only replacing the : with underscore so
# that we can use ImageReader class to help us later on.
if ":" in self.image_name:
logger.info(f"Looking for {self.image_name}...")
# e.g. fastqc:0.11.9
registry_name = self.image_name #.replace(":", "_")
if self.image_name not in self.registry.registry.keys():
logger.critical(
"invalid image name provided: {}. type 'damona list'".
format(self.image_name))
guess = self.image_name.split(":")[0][0:4]
# here we reverse x to start from the end and replace the last _
# by :
guesses = [
x[::-1].replace("_", ":", 1)[::-1]
for x in self.registry.registry.keys()
if x.startswith(guess)
]
logger.critical("Maybe you meant one of: {}".format(guesses))
sys.exit(1)
else:
from damona.registry import Software
r = Software(self.image_name).releases
latest = r.last_release
logger.info(
f"No version found after {self.image_name} (e.g. fastqc:0.11.8)."
+ f" Installing latest version {latest}")
registry_name = self.image_name
# we look only at the prefix name, not the tag. so we should get the
# registry names from self.registry that have the prefix in common,
# then extract the versions, and figure out the most recent.
registry_name = self.registry.find_candidate(registry_name)
if registry_name is None:
logger.critical(
f"No image found for {self.image_name}. Make sure it is correct using 'damona search' command"
)
sys.exit(1)
download_name = self.registry.registry[registry_name].download
# get metadata from the registry recipe
info = self.registry.registry[registry_name]
logger.info(f"{info}")
if info.binaries:
self.binaries = info.binaries
else:
logger.warning(
f"No binaries field found in registry of {registry_name}")
self.binaries = [self.input_image.guessed_executable]
# target file is stored in output_name variable
if output_name is None:
output_name = registry_name.replace(":", "_") + ".img"
# here we check whether the image or binaries are already present.
# if md5 is already provided, and image exists, nothing to copy
if info.md5sum:
if os.path.exists(self.images_directory / output_name):
from easydev import md5
md5_target = md5(self.images_directory / output_name)
if md5_target == info.md5sum:
logger.info(
"Remote image and local image are identical, no need to download/pull again"
)
self.input_image = ImageReader(self.images_directory /
output_name)
self.image_installed = True
return
else:
logger.warning(
f"md5 field not found or not filled in {registry_name}."
f"To be fixed in https://github.com/damona/damona/recipes/{self.image_name}/registry.yaml "
)
# now that we have the registry name, we can download the image
logger.info("Downloading {}".format(download_name))
# By default we downlaods from syslab.
# if not found, one can provide an URL
pull_folder = self.images_directory / "damona_buffer"
if self.from_url:
# The client does not support external https link other than
# docker, library, shub.
cmd = f"singularity pull --dir {pull_folder} "
if force:
cmd += " --force "
cmd += f"{download_name}"
subprocess.call(cmd.split())
else:
if download_name.startswith("https://"):
print(f"downloading into {pull_folder} {output_name}")
from urllib.request import urlretrieve
urlretrieve(download_name,
filename=str(pull_folder / output_name))
# wget.download(download_name, str(pull_folder / output_name))
else: # use singularity
Client.pull(str(download_name),
name=output_name,
pull_folder=pull_folder,
force=force)
logger.info(f"File {self.image_name} uploaded to {pull_folder}")
# Read the image, checking everything is correct
self.input_image = ImageReader(pull_folder / output_name)
shortname = self.input_image.shortname
try:
logger.info(
f"Copying into damona image directory: {self.images_directory}"
)
self.input_image.filename.rename(self.images_directory / shortname)
self.input_image.filename = pathlib.Path(self.images_directory /
shortname)
except FileNotFoundError:
logger.warning("File not installed properly. Stopping")
self.image_installed = False
# check the md5 validity
if info.md5sum:
if info.md5sum != self.input_image.md5:
logger.warning(
"MD5 of downloaded image does not match the expected md5 found in the "
f"registry of {registry_name}. The latter may be incorrect in damona and needs "
f"to be updated in https://github.com/damona/damona/recipes/{self.image_name}/registry.yaml "
"or the donwload was interrupted")
self.image_installed = True
read_file,
write_file
)
from spython.main import Client
import unittest
import tempfile
import shutil
import json
import os
print("######################################################## test_reproduce")
# Pull images for all tests
tmpdir = tempfile.mkdtemp()
image1 = Client.pull('docker://ubuntu:14.04', pull_folder=tmpdir)
image2 = Client.pull('docker://busybox:1', pull_folder=tmpdir)
class TestReproduce(unittest.TestCase):
def setUp(self):
self.pwd = get_installdir()
self.tmpdir = tmpdir
self.image1 = image1
self.image2 = image2
def tearDown(self):
pass
def test_get_image_hashes(self):
from singularity.analysis.reproduce import get_image_hashes, get_image_hash
def _pull_image(self):
client.pull(image=self.img_url,
name=self.img_name,
pull_folder=self.img_dir)
return
def run_singularity_container_command(
uuid,
container_image,
command_to_run,
logs_path,
results_path,
env_vars_list,
args_dict,
):
"""Launch an executable within a Singularity container.
Args:
uuid: A string containing the uuid of the job being run.
container_image: A string containing the name of the container
to pull.
command_to_run: A string containing the command to run.
logs_path: A string (or None) containing the path of the
directory containing the relevant logs within the container.
results_path: A string (or None) containing the path of the
directory containing any output files from the container.
env_vars_list: A list of strings containing the environment
variable names for the worker to consume from its
environment.
args_dict: A dictionary containing arguments and corresponding
values.
Raises:
KeyError: An environment variable specified was not available in
the worker's environment.
SingularityPullFailure: The Singularity pull could not complete
with the specified timeout and number of retries.
"""
# Import Singularity library
from spython.main import Client as client
# Pull the specified container. This pull in the latest version of
# the container (with the specified tag if provided).
timeout = int(os.environ["SINGULARITY_PULL_TIMEOUT"])
num_retries = int(os.environ["SINGULARITY_PULL_RETRIES"])
# Put a timeout on the client pull method
client.pull = timeout_decorator.timeout(timeout,
timeout_exception=StopIteration)(
client.pull)
for retry in range(num_retries):
try:
singularity_image = client.pull(
image=container_image,
pull_folder=os.environ["WORKER_SINGULARITY_IMAGES_DIRECTORY"],
name_by_commit=True,
)
break
except StopIteration:
# If this is the last retry, raise an exception to indicate
# a failed job
if retry == num_retries - 1:
raise SingularityPullFailure(
("Could not pull {image_url} within "
"{timeout} seconds after {num_retries} retries.").format(
image_url=container_image,
timeout=timeout,
num_retries=num_retries,
))
# Find out where to put the logs
if logs_path is None:
bind_option = []
else:
# Create the host logs path. This is required by the Singularity
# library (though not the Docker library)
host_logs_path = os.path.join(os.environ["WORKER_LOGS_DIRECTORY"],
uuid)
create_local_directory(host_logs_path)
# Build the bind option to pass on to Singularity
bind_option = [
host_logs_path.rstrip("/") + ":" + logs_path.rstrip("/")
]
# Find out where to put the results
if results_path is not None:
# Create the host results path
host_results_path = os.path.join(
os.environ["WORKER_RESULTS_DIRECTORY"], uuid)
create_local_directory(host_results_path)
# Build the bind option to pass on to Singularity
bind_option += [
host_results_path.rstrip("/") + ":" + results_path.rstrip("/")
]
# Check for required environment variables. Note that by default
# Singularity containers have access to their outside environment
# variables, so we don't need to pass them along explicitly like we
# need to for a Docker container.
try:
# Test to see that all keys are defined
{key: os.environ[key] for key in env_vars_list}
except KeyError as e:
raise KeyError(
"Environment variable %s not present in the worker's environment!"
% e)
# Pass along the job's UUID
os.environ["JOB_UUID"] = uuid
# Compose the command to run
command = shlex.split(command_to_run)
if args_dict:
command += [json.dumps(args_dict)]
# Run the executable
iter_ = client.execute(image=singularity_image,
command=command,
bind=bind_option,
stream=True)
# Okay, here's some magic. The issue is that without stream=True in
# the above call, there's no way of determining the return code of
# the above operation, and so no way of knowing whether it failed.
# However, with stream=True, it'll raise a
# subprocess.CalledProcessError exception for any non-zero return
# code. Great! But before we can get that exception triggered we
# need to iterate through all of the command's stdout, which is what
# the below (seemingly useless) loop does.
for _ in iter_:
pass
def singularity_pull(self, image):
"""Pulls an docker or singularity images from hub.
"""
Client.pull(image)
def main():
'''main is the entrypoint to run a container comparison.
'''
parser = get_parser()
try:
args = parser.parse_args()
except:
sys.exit(0)
if args.containers in [None, '', []]:
print('Please provide a list of one or more containers to compare.')
# Step 1: Extraction of files
print('1. Staring extraction for %s containers.' % len(args.containers))
# We will save lists of files for comparison
data = []
for container in args.containers:
# if the file exists, assume it's singularity
func = run_analyze
if os.path.exists(container):
print('Found Singularity container file %s' % container)
# If it starts with shub:// or docker:// we want to singularity pull
if re.search('^(docker|shub)[://]', container):
print('Pulling Singularity container %s' % container)
container = Client.pull(container, pull_folder='/tmp')
# Otherwise, must be docker container
else:
func = run_container_diff
# Prefix of container for output
name = os.path.basename(container)
# Just generate files
output_files = os.path.join(output, '%s-files.json' % name)
# Run the analyze-singularity.sh for each of files and packages
if not os.path.exists(output_files):
print("Performing Extraction for %s" % container)
dest = func(container, dest=output_files)
else:
dest = output_files
print(dest)
data.append(dest)
# Step 2: Generation of Web Interfaces
print('2. Calculating comparisons')
scores = pandas.DataFrame()
# Lookup for files
lookup = dict()
htmls = dict()
for d in data:
datum = json.load(open(d, 'r'))
osname = os.path.basename(d).replace('-files.json', '')
files = []
for f in datum[0]['Analysis']:
files.append(f['Name'])
lookup[osname] = files
# Now calculate differences (and html tree) for each.
for name1, files1 in lookup.items():
for name2, files2 in lookup.items():
# browser can have trouble with weird characters
name = ("%s-%s" % (name1, name2)).replace(':',
'').replace('/', '-')
# Calculate the score in matrix
comparison = compare_lists(files1, files2)
score = compare_files(files1, files2)
scores.loc[name1, name2] = score
# Create labels lookup to show which were added and removed
labels = dict()
allfiles = set(files1).union(set(files2))
for afile in allfiles:
if afile in comparison['shared']: labels[afile] = 'shared'
elif afile in comparison['added']: labels[afile] = 'added'
elif afile in comparison['removed']: labels[afile] = 'removed'
# Generate tree that shows added and subtracted nodes
tree = make_container_tree(allfiles, labels=labels)
html = get_template(
'container_tree', {
'{{ files | safe }}': json.dumps(tree['files']),
'{{ graph | safe }}': json.dumps(tree['graph']),
'{{ container_name }}': "%s --> %s Tree" % (name1, name2)
})
html_file = '%s/%s.html' % (output, name)
htmls['%s vs. %s' % (name1, name2)] = html_file
with open(html_file, 'w') as filey:
filey.writelines(html)
# Create web plot
values = ',\n'.join(
[str(values.tolist()) for row, values in scores.items()])
# Write list of html files
extra = '<br><div><h2>View Comparison</h2><ul>'
for name, html_file in htmls.items():
extra += '<li><a href="%s">%s</a></li>\n' % (
os.path.basename(html_file), name)
extra += '</ul></div></body>'
# Plot via html
template = get_template(
'heatmap', {
'{{ title }}': "Similarity of Containers based on Filesystem",
'{{ data }}': values,
'{{ X }}': str(scores.index.tolist()),
'{{ Y }}': str(scores.columns.tolist()),
'</body>': extra
})
# Save scores for user as data frame
scores.to_csv('%s/information-coefficient-scores.tsv' % output, sep='\t')
with open('%s/index.html' % output, 'w') as filey:
filey.writelines(template)
# Open browser there
os.chdir(output)
Handler = http.server.SimpleHTTPRequestHandler
with socketserver.TCPServer(("", 8888), Handler) as httpd:
print("Open browser to http://0.0.0.0:8888")
httpd.serve_forever()
def docker_container(tmp_path_factory):
folder = tmp_path_factory.mktemp("docker-img")
return folder, Client.pull("docker://busybox:1.30.1",
pull_folder=str(folder))
def _pull(self, file_name, names, save=True, force=False, **kwargs):
'''pull an image from aws. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers from S3
- converts to a squashfs image with build
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(names)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.error('Image exists! Remove first, or use --force to overwrite')
sys.exit(1)
digest = names['version'] or names['tag']
# Build from sandbox
sandbox = get_tmpdir(prefix="sregistry-sandbox")
# First effort, get image via Sregistry
layers, url = self._download_layers(names['url'], digest)
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info('Exploding %s' % layer)
result = extract_tar(layer, sandbox, handle_whiteout=True)
if result['return_code'] != 0:
bot.error(result['message'])
sys.exit(1)
sudo = kwargs.get('sudo', False)
# Build from a sandbox (recipe) into the image_file (squashfs)
image_file = Singularity.build(image=file_name, recipe=sandbox, sudo=sudo)
# Fall back to using Singularity
if image_file is None:
bot.info('Downloading with native Singularity, please wait...')
image = image.replace('aws://', 'docker://')
image_file = Singularity.pull(image, pull_folder=sandbox)
# Save to local storage
if save is True:
# Did we get the manifests?
manifests = {}
if hasattr(self, 'manifest'):
manifest = self.manifest
container = self.add(image_path=image_file,
image_uri=names['uri'],
metadata=manifest,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
# Clean up sandbox
shutil.rmtree(sandbox)
return image_file
