def get_layer(self, image_id, repo_name, download_folder=None):
'''download an image layer (.tar.gz) to a specified download folder.
Parameters
==========
download_folder: download to this folder. If not set, uses temp.
repo_name: the image name (library/ubuntu) to retrieve
'''
url = self._get_layerLink(repo_name, image_id)
bot.verbose("Downloading layers from %s" % url)
download_folder = get_tmpdir(download_folder)
download_folder = "%s/%s.tar.gz" % (download_folder, image_id)
# Update user what we are doing
bot.debug("Downloading layer %s" % image_id)
# Step 1: Download the layer atomically
file_name = "%s.%s" % (download_folder,
next(tempfile._get_candidate_names()))
tar_download = self.download(url, file_name)
try:
shutil.move(tar_download, download_folder)
except Exception:
msg = "Cannot untar layer %s," % tar_download
msg += " was there a problem with download?"
bot.error(msg)
sys.exit(1)
return download_folder
def create_build_package(package_files, working_dir=None):
"""given a list of files, copy them to a temporary folder,
compress into a .tar.gz, and rename based on the file hash.
Return the full path to the .tar.gz in the temporary folder.
Parameters
==========
package_files: a list of files to include in the tar.gz
working_dir: If set, the path derived for the recipe and
files is relative to this.
"""
# Ensure package files all exist
for package_file in package_files:
if not os.path.exists(package_file):
bot.exit("Cannot find %s." % package_file)
bot.log("Generating build package for %s files..." % len(package_files))
build_dir = get_tmpdir(prefix="sregistry-build")
build_tar = "%s/build.tar.gz" % build_dir
tar = tarfile.open(build_tar, "w:gz")
# Create the tar.gz, making sure relative to working_dir
for package_file in package_files:
# Get a relative path
relative_path = get_relative_path(package_file, working_dir)
tar.add(package_file, arcname=relative_path)
tar.close()
# Get hash (sha256), and rename file
sha256 = get_file_hash(build_tar)
hash_tar = "%s/%s.tar.gz" % (build_dir, sha256)
shutil.move(build_tar, hash_tar)
return hash_tar
def create_build_package(package_files):
'''given a list of files, copy them to a temporary folder,
compress into a .tar.gz, and rename based on the file hash.
Return the full path to the .tar.gz in the temporary folder.
Parameters
==========
package_files: a list of files to include in the tar.gz
'''
# Ensure package files all exist
for package_file in package_files:
if not os.path.exists(package_file):
bot.exit('Cannot find %s.' % package_file)
bot.log('Generating build package for %s files...' % len(package_files))
build_dir = get_tmpdir(prefix="sregistry-build")
build_tar = '%s/build.tar.gz' % build_dir
tar = tarfile.open(build_tar, "w:gz")
# Create the tar.gz
for package_file in package_files:
tar.add(package_file)
tar.close()
# Get hash (sha256), and rename file
sha256 = get_file_hash(build_tar)
hash_tar = "%s/%s.tar.gz" % (build_dir, sha256)
shutil.move(build_tar, hash_tar)
return hash_tar
def test_get_tmpdir_tmpfile():
print("Testing utils.get_tmpdir, get_tmpfile")
from sregistry.utils import get_tmpdir, get_tmpfile
tmpdir = get_tmpdir()
assert os.path.exists(tmpdir)
assert os.path.basename(tmpdir).startswith('sregistry')
shutil.rmtree(tmpdir)
tmpdir = get_tmpdir(prefix='name')
assert os.path.basename(tmpdir).startswith('name')
shutil.rmtree(tmpdir)
tmpfile = get_tmpfile()
assert 'sregistry' in tmpfile
os.remove(tmpfile)
tmpfile = get_tmpfile(prefix="pancakes")
assert 'pancakes' in tmpfile
os.remove(tmpfile)
def get_download_cache(self, destination, subfolder="docker"):
"""determine the user preference for atomic download of layers. If
the user has set a singularity cache directory, honor it. Otherwise,
use the Singularity default.
"""
# First priority after user specification is Singularity Cache
if destination is None:
destination = self._get_setting("SINGULARITY_CACHEDIR", SINGULARITY_CACHE)
# If not set, the user has disabled (use tmp)
destination = get_tmpdir(destination)
if not destination.endswith(subfolder):
destination = "%s/%s" % (destination, subfolder)
# Create subfolders, if don't exist
mkdir_p(destination)
return destination
def _pull(self,
file_name,
names,
save=True,
force=False,
uri="docker://",
**kwargs):
"""pull an image from a docker hub. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers, metadata folder, and custom metadata to it
- converts to a squashfs image with build
the docker manifests are stored with registry metadata.
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
"""
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(names)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.exit("Image exists! Remove first, or use --force to overwrite")
digest = names["version"] or names["tag"]
# Build from sandbox, prefix with sandbox
sandbox = get_tmpdir(prefix="sregistry-sandbox")
# First effort, get image via Sregistry
layers = self._download_layers(names["url"], digest)
# This is the url where the manifests were obtained
url = self._get_manifest_selfLink(names["url"], digest)
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info("Exploding %s" % layer)
result = extract_tar(layer, sandbox, handle_whiteout=True)
if result["return_code"] != 0:
bot.exit(result["message"])
sudo = kwargs.get("sudo", False)
# Build from a sandbox (recipe) into the image_file (squashfs)
image_file = Singularity.build(image=file_name, recipe=sandbox, sudo=sudo)
# Fall back to using Singularity
if image_file is None:
bot.info("Downloading with native Singularity, please wait...")
image = file_name.replace("docker://", uri)
image_file = Singularity.pull(image, pull_folder=sandbox)
# Save to local storage
if save is True:
# Did we get the manifests?
manifests = {}
if hasattr(self, "manifests"):
manifests = self.manifests
container = self.add(image_path=image_file,
image_uri=names["uri"],
metadata=manifests,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug("Retrieved image file %s" % image_file)
bot.custom(prefix="Success!", message=image_file)
# Clean up sandbox
shutil.rmtree(sandbox)
return image_file
def _pull(self, file_name, names, save=True, force=False, **kwargs):
'''pull an image from aws. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers from S3
- converts to a squashfs image with build
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(names)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.error('Image exists! Remove first, or use --force to overwrite')
sys.exit(1)
digest = names['version'] or names['tag']
# Build from sandbox
sandbox = get_tmpdir(prefix="sregistry-sandbox")
# First effort, get image via Sregistry
layers, url = self._download_layers(names['url'], digest)
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info('Exploding %s' % layer)
result = extract_tar(layer, sandbox, handle_whiteout=True)
if result['return_code'] != 0:
bot.error(result['message'])
sys.exit(1)
sudo = kwargs.get('sudo', False)
# Build from a sandbox (recipe) into the image_file (squashfs)
image_file = Singularity.build(image=file_name, recipe=sandbox, sudo=sudo)
# Fall back to using Singularity
if image_file is None:
bot.info('Downloading with native Singularity, please wait...')
image = image.replace('aws://', 'docker://')
image_file = Singularity.pull(image, pull_folder=sandbox)
# Save to local storage
if save is True:
# Did we get the manifests?
manifests = {}
if hasattr(self, 'manifest'):
manifest = self.manifest
container = self.add(image_path=image_file,
image_uri=names['uri'],
metadata=manifest,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
# Clean up sandbox
shutil.rmtree(sandbox)
return image_file
