def test_download_image(self):
'''test_download_image will ensure that an image is downloaded to an
appropriate location (tmpdir) or cache
'''
from shub.api import download_image, get_manifest
print("Case 1: Specifying a directory downloads to it")
manifest = get_manifest(image=self.image_id)
image = download_image(manifest, download_folder=self.tmpdir)
self.assertEqual(os.path.dirname(image), self.tmpdir)
print("Case 2: Image should be named based on commit.")
image_name = os.path.splitext(os.path.basename(image))[0]
self.assertEqual(image_name, manifest['version'])
os.remove(image)
print("Case 3: Not specifying a directory downloads to PWD")
os.chdir(self.tmpdir)
image = download_image(manifest)
self.assertEqual(os.getcwd(), self.tmpdir)
self.assertTrue(image in glob("*"))
os.remove(image)
print("Case 4: Image should not be extracted.")
image = download_image(manifest, extract=False)
self.assertTrue(image.endswith('.img.gz'))
def test_download_image(self):
'''test_download_image will ensure that an image is downloaded to an
appropriate location (tmpdir) or cache
'''
from shub.api import download_image, get_manifest
print("Case 1: Specifying a directory downloads to it")
manifest = get_manifest(image_id=self.image_id)
image = download_image(manifest,
download_folder=self.tmpdir)
self.assertEqual(os.path.dirname(image),self.tmpdir)
os.remove(image)
print("Case 2: Not specifying a directory downloads to PWD")
os.chdir(self.tmpdir)
image = download_image(manifest)
self.assertEqual(os.getcwd(),self.tmpdir)
os.remove(image)
print("Case 3: Image should not be extracted.")
image = download_image(manifest,extract=False)
self.assertTrue(image.endswith('.img.gz'))
def run(args):
# Find root filesystem location
if args.rootfs != None:
singularity_rootfs = args.rootfs
else:
singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
if singularity_rootfs == None and args.shub == None:
logger.error(
"root file system not specified OR defined as environmental variable, exiting!"
)
sys.exit(1)
if singularity_rootfs != None:
logger.info("Root file system defined as %s", singularity_rootfs)
# Does the registry require authentication?
auth = None
if args.username is not None and args.password is not None:
auth = basic_auth_header(args.username, args.password)
logger.info("Username for registry authentication: %s", args.username)
# Does the user want to download a Singularity image?
if args.shub != None:
image = args.shub
manifest = get_shub_manifest(image)
if args.pull_folder == None:
cache_base = get_cache(subfolder="shub",
disable_cache=args.disable_cache)
else:
cache_base = args.pull_folder
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
image_file = "%s/%s" % (cache_base, image_name)
if not os.path.exists(image_file):
image_file = download_image(manifest=manifest,
download_folder=cache_base)
else:
print("Image already exists at %s, skipping download." %
image_file)
logger.info("Singularity Hub Image Download: %s", image_file)
# If singularity_rootfs is provided, write metadata to it
if singularity_rootfs != None:
logger.debug(
"Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",
singularity_rootfs)
write_file("%s/SINGULARITY_RUNDIR" % singularity_rootfs,
os.path.dirname(image_file))
write_file("%s/SINGULARITY_IMAGE" % singularity_rootfs, image_file)
# Do we have a docker image specified?
elif args.docker != None:
# Does the user want to override default Entrypoint and use CMD as runscript?
includecmd = args.includecmd
logger.info("Including Docker command as Runscript? %s", includecmd)
image = args.docker
logger.info("Docker image: %s", image)
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
image = parse_image_uri(image=image, uri="docker://")
namespace = image['namespace']
repo_name = image['repo_name']
repo_tag = image['repo_tag']
# Tell the user the namespace, repo name and tag
logger.info("Docker image path: %s/%s:%s", namespace, repo_name,
repo_tag)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
manifest = get_manifest(repo_name=repo_name,
namespace=namespace,
repo_tag=repo_tag,
registry=args.registry,
auth=auth)
# Get images from manifest using version 2.0 of Docker Registry API
images = get_images(manifest=manifest)
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker",
disable_cache=args.disable_cache)
layers = []
for image_id in images:
# Download the layer, if we don't have it
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
targz = get_layer(image_id=image_id,
namespace=namespace,
repo_name=repo_name,
download_folder=cache_base,
registry=args.registry,
auth=auth)
layers.append(targz) # in case we want a list at the end
# Extract image and remove tar
output = extract_tar(targz, singularity_rootfs)
if output is None:
logger.error("Error extracting image: %s", targz)
sys.exit(1)
if args.disable_cache == True:
os.remove(targz)
# If the user wants to include the CMD as runscript, generate it here
if includecmd == True:
spec = "Cmd"
else:
spec = "Entrypoint"
cmd = get_config(manifest, spec=spec)
# Only add runscript if command is defined
if cmd != None:
print("Adding Docker %s as Singularity runscript..." %
(spec.upper()))
print(cmd)
runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)
# When we finish, clean up images
if args.disable_cache == True:
shutil.rmtree(cache_base)
logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def run(args):
# Find root filesystem location
if args.rootfs != None:
singularity_rootfs = args.rootfs
else:
singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
if singularity_rootfs == None and args.shub == None:
logger.error("root file system not specified OR defined as environmental variable, exiting!")
sys.exit(1)
if singularity_rootfs != None:
logger.info("Root file system defined as %s", singularity_rootfs)
# Does the registry require authentication?
auth = None
if args.username is not None and args.password is not None:
auth = basic_auth_header(args.username, args.password)
logger.info("Username for registry authentication: %s", args.username)
# Does the user want to download a Singularity image?
if args.shub != None:
image_id = int(args.shub)
manifest = get_shub_manifest(image_id)
cache_base = get_cache(subfolder="shub",
disable_cache = args.disable_cache)
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
image_file = "%s/%s" %(cache_base,image_name)
if not os.path.exists(image_file):
image_file = download_image(manifest=manifest,
download_folder=cache_base)
else:
print("Image already exists at %s, skipping download." %image_file)
logger.info("Singularity Hub Image Download: %s", image_file)
# If singularity_rootfs is provided, write metadata to it
if singularity_rootfs != None:
logger.debug("Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",singularity_rootfs)
write_file("%s/SINGULARITY_RUNDIR" %singularity_rootfs, os.path.dirname(image_file))
write_file("%s/SINGULARITY_IMAGE" %singularity_rootfs, image_file)
# Do we have a docker image specified?
elif args.docker != None:
# Does the user want to override default Entrypoint and use CMD as runscript?
includecmd = args.includecmd
logger.info("Including Docker command as Runscript? %s", includecmd)
image = args.docker
logger.info("Docker image: %s", image)
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
# First split the docker image name by /
image = image.split('/')
# If there are two parts, we have namespace with repo (and maybe tab)
if len(image) == 2:
namespace = image[0]
image = image[1]
# Otherwise, we must be using library namespace
else:
namespace = "library"
image = image[0]
# Now split the docker image name by :
image = image.split(':')
if len(image) == 2:
repo_name = image[0]
repo_tag = image[1]
# Otherwise, assume latest of an image
else:
repo_name = image[0]
repo_tag = "latest"
# Tell the user the namespace, repo name and tag
logger.info("Docker image path: %s/%s:%s", namespace,repo_name,repo_tag)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
manifest = get_manifest(repo_name=repo_name,
namespace=namespace,
repo_tag=repo_tag,
registry=args.registry,
auth=auth)
# Get images from manifest using version 2.0 of Docker Registry API
images = get_images(repo_name=repo_name,
namespace=namespace,
registry=args.registry,
auth=auth)
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker",
disable_cache = args.disable_cache)
layers = []
for image_id in images:
# Download the layer, if we don't have it
targz = "%s/%s.tar.gz" %(cache_base,image_id)
if not os.path.exists(targz):
targz = get_layer(image_id=image_id,
namespace=namespace,
repo_name=repo_name,
download_folder=cache_base,
registry=args.registry,
auth=auth)
layers.append(targz) # in case we want a list at the end
# Extract image and remove tar
output = extract_tar(targz,singularity_rootfs)
if output is None:
logger.error("Error extracting image: %s", targz)
sys.exit(1)
if args.disable_cache == True:
os.remove(targz)
# If the user wants to include the CMD as runscript, generate it here
if includecmd == True:
spec="Cmd"
else:
spec="Entrypoint"
cmd = get_config(manifest,spec=spec)
# Only add runscript if command is defined
if cmd != None:
print("Adding Docker %s as Singularity runscript..." %(spec.upper()))
print(cmd)
runscript = create_runscript(cmd=cmd,
base_dir=singularity_rootfs)
# When we finish, clean up images
if args.disable_cache == True:
shutil.rmtree(cache_base)
logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")