def test_read_tar(tmp_path):
print("Testing utils.extract_tar")
from sregistry.utils import extract_tar
here = os.path.dirname(os.path.abspath(__file__))
testdata = os.path.join(here, 'testdata', 'hashtest.tar.gz')
dirname = str(tmp_path / "destination")
os.mkdir(dirname)
extract_tar(testdata, dirname)
assert os.path.exists(dirname)
assert "hashtest.txt" in [os.path.basename(x) for x in os.listdir(dirname)]
def _pull(self,
file_name,
names,
save=True,
force=False,
uri="docker://",
**kwargs):
"""pull an image from a docker hub. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers, metadata folder, and custom metadata to it
- converts to a squashfs image with build
the docker manifests are stored with registry metadata.
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
"""
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(names)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.exit("Image exists! Remove first, or use --force to overwrite")
digest = names["version"] or names["tag"]
# Build from sandbox, prefix with sandbox
sandbox = get_tmpdir(prefix="sregistry-sandbox")
# First effort, get image via Sregistry
layers = self._download_layers(names["url"], digest)
# This is the url where the manifests were obtained
url = self._get_manifest_selfLink(names["url"], digest)
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info("Exploding %s" % layer)
result = extract_tar(layer, sandbox, handle_whiteout=True)
if result["return_code"] != 0:
bot.exit(result["message"])
sudo = kwargs.get("sudo", False)
# Build from a sandbox (recipe) into the image_file (squashfs)
image_file = Singularity.build(image=file_name, recipe=sandbox, sudo=sudo)
# Fall back to using Singularity
if image_file is None:
bot.info("Downloading with native Singularity, please wait...")
image = file_name.replace("docker://", uri)
image_file = Singularity.pull(image, pull_folder=sandbox)
# Save to local storage
if save is True:
# Did we get the manifests?
manifests = {}
if hasattr(self, "manifests"):
manifests = self.manifests
container = self.add(image_path=image_file,
image_uri=names["uri"],
metadata=manifests,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug("Retrieved image file %s" % image_file)
bot.custom(prefix="Success!", message=image_file)
# Clean up sandbox
shutil.rmtree(sandbox)
return image_file
def _pull(self, file_name, names, save=True, force=False, **kwargs):
'''pull an image from aws. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers from S3
- converts to a squashfs image with build
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(names)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.error('Image exists! Remove first, or use --force to overwrite')
sys.exit(1)
digest = names['version'] or names['tag']
# Build from sandbox
sandbox = get_tmpdir(prefix="sregistry-sandbox")
# First effort, get image via Sregistry
layers, url = self._download_layers(names['url'], digest)
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info('Exploding %s' % layer)
result = extract_tar(layer, sandbox, handle_whiteout=True)
if result['return_code'] != 0:
bot.error(result['message'])
sys.exit(1)
sudo = kwargs.get('sudo', False)
# Build from a sandbox (recipe) into the image_file (squashfs)
image_file = Singularity.build(image=file_name, recipe=sandbox, sudo=sudo)
# Fall back to using Singularity
if image_file is None:
bot.info('Downloading with native Singularity, please wait...')
image = image.replace('aws://', 'docker://')
image_file = Singularity.pull(image, pull_folder=sandbox)
# Save to local storage
if save is True:
# Did we get the manifests?
manifests = {}
if hasattr(self, 'manifest'):
manifest = self.manifest
container = self.add(image_path=image_file,
image_uri=names['uri'],
metadata=manifest,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
# Clean up sandbox
shutil.rmtree(sandbox)
return image_file
def pull(self,
images,
file_name=None,
save=True,
force=False,
base=None,
**kwargs):
'''pull an image from a docker hub. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers, metadata folder, and custom metadata to it
- converts to a squashfs image with build
the docker manifests are stored with registry metadata.
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
base: the registry base, in case the client doesn't want to set in env.
Returns
=======
finished: a single container path, or list of paths
'''
if not isinstance(images, list):
images = [images]
bot.debug('Execution of PULL for %s images' % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
# 0. Update the base in case we aren't working with default
base = self._update_base(image)
q = parse_image_name(remove_uri(image), base=base)
digest = q['version'] or q['tag']
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(q)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.error(
'Image exists! Remove first, or use --force to overwrite')
sys.exit(1)
# This is the url where the manifests are obtained
url = self._get_manifest_selfLink(q['url'], digest)
# This is the Docker Hub namespace and repository
layers = self._download_layers(q['url'], digest)
# Create client to build from sandbox
cli = Singularity()
# Build from sandbox
sandbox = tempfile.mkdtemp()
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info('Exploding %s' % layer)
result = extract_tar(layer, sandbox)
if result['return_code'] != 0:
bot.error(result['message'])
sys.exit(1)
if os.geteuid() == 0:
image_file = cli.build(file_name, sandbox)
else:
image_file = cli.build(file_name, sandbox, sudo=False)
# Save to local storage
if save is True:
container = self.add(image_path=image_file,
image_uri=q['uri'],
metadata=self.manifests,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
# If the image_file is different from sandbox, remove sandbox
if image_file != sandbox:
shutil.rmtree(sandbox)
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished