def test_uri(self):
'''test_uri will make sure that the endpoint returns the equivalent
image for all different uri options
'''
from shub.api import get_image_name, get_manifest
manifest = get_manifest(image=self.image_id)
image_name = get_image_name(manifest)
print("Case 1: ask for image and ask for master branch (tag)")
manifest = get_manifest(image="%s/%s:master" %
(self.user_name, self.repo_name))
self.assertEqual(image_name, get_image_name(manifest))
print("Case 2: ask for different tag (mongo)")
manifest = get_manifest(image="%s/%s:mongo" %
(self.user_name, self.repo_name))
mongo = get_image_name(manifest)
self.assertFalse(image_name == mongo)
print(
"Case 3: ask for image without tag (should be latest across tags, mongo)"
)
manifest = get_manifest(image="%s/%s" %
(self.user_name, self.repo_name))
self.assertEqual(mongo, get_image_name(manifest))
print(
"Case 4: ask for latest tag (should be latest across tags, mongo)")
manifest = get_manifest(image="%s/%s:latest" %
(self.user_name, self.repo_name))
self.assertEqual(mongo, get_image_name(manifest))
def test_uri(self):
'''test_uri will make sure that the endpoint returns the equivalent
image for all different uri options
'''
from shub.api import get_image_name
manifest = self.client.get_manifest()
image_name = get_image_name(manifest)
fullname = "%s/%s" % (self.user_name, self.repo_name)
print("Case 1: ask for image by tag.")
client = self.connect(image="%s:latest" % fullname)
manifest = client.get_manifest()
image_name = get_image_name(manifest)
self.assertEqual(image_name, get_image_name(manifest))
def test_uri(self):
'''test_uri will make sure that the endpoint returns the equivalent
image for all different uri options
'''
print("Testing singularity hub uris")
from shub.api import get_image_name
manifest = self.client.get_manifest()
image_name = get_image_name(manifest)
fullname = "%s/%s" % (self.user_name, self.repo_name)
print("...case 1: ask for image by tag.")
client = self.connect(image="%s:latest" % fullname)
manifest = client.get_manifest()
image_name = get_image_name(manifest)
self.assertEqual(image_name, get_image_name(manifest))
def test_get_image_name(self):
'''test_get_image_name will return the image name from the manifest
'''
from shub.api import get_image_name
manifest = self.client.get_manifest()
print("Case 1: return an image name corresponding to repo")
image_name = get_image_name(manifest)
self.assertEqual('vsoch-singularity-images-mongo.img.gz', image_name)
def test_get_image_name(self):
'''test_get_image_name will return the image name from the manifest
'''
print("Testing singularity hub image naming")
from shub.api import get_image_name
manifest = self.client.get_manifest()
print("...case 1: return an image name corresponding to repo")
image_name = get_image_name(manifest)
self.assertEqual('vsoch-singularity-images-master-latest', image_name)
def test_get_image_name(self):
'''test_get_image_name will return the image name from the manifest
'''
from shub.api import get_image_name
manifest = self.client.get_manifest(image=self.image_id)
print("Case 1: return an image name corresponding to repo")
image_name = get_image_name(manifest)
self.assertEqual('vsoch-singularity-images-master.img.gz',
image_name)
def test_get_image_name(self):
'''test_get_image_name will return the image name from the manifest
'''
from shub.api import get_image_name, get_manifest
manifest = get_manifest(image_id=self.image_id)
print("Case 1: return an image name using the commit id")
image_name = get_image_name(manifest)
self.assertEqual('f57e631a0434c31f0b4fa5276a314a6d8a672a55.img.gz',
image_name)
print("Case 2: ask for invalid extension")
with self.assertRaises(SystemExit) as cm:
image_name = get_image_name(manifest, extension='.bz2')
self.assertEqual(cm.exception.code, 1)
print("Case 3: don't use commit (use md5 sum on generation)")
image_name = get_image_name(manifest, use_commit=False)
print(image_name)
self.assertEqual('be4b9ba8fc22525d2ee2b27846513d42.img.gz', image_name)
def test_get_image_name(self):
'''test_get_image_name will return the image name from the manifest
'''
from shub.api import get_image_name, get_manifest
manifest = get_manifest(image=self.image_id)
print("Case 1: return an image name using the commit id")
image_name = get_image_name(manifest)
self.assertEqual('6d3715a982865863ff20e8783014522edf1240e4.img.gz',
image_name)
print("Case 2: ask for invalid extension")
with self.assertRaises(SystemExit) as cm:
image_name = get_image_name(manifest, extension='.bz2')
self.assertEqual(cm.exception.code, 1)
print("Case 3: don't use commit (use md5 sum on generation)")
image_name = get_image_name(manifest, use_commit=False)
print(image_name)
self.assertEqual('9e46ba8be1e10b1a2812844ac8072259.img.gz', image_name)
def test_uri(self):
'''test_uri will make sure that the endpoint returns the equivalent
image for all different uri options
'''
from shub.api import get_image_name
manifest = self.client.get_manifest()
image_name = get_image_name(manifest)
print("Case 1: ask for image and ask for master branch (tag)")
manifest = self.client.get_manifest(image="%s/%s:master" %(self.user_name,self.repo_name))
image_name = get_image_name(manifest)
self.assertEqual(image_name,get_image_name(manifest))
print("Case 2: ask for different tag (mongo)")
manifest = self.client.get_manifest(image="%s/%s:mongo" %(self.user_name,self.repo_name))
mongo = get_image_name(manifest)
self.assertFalse(image_name == mongo)
print("Case 3: ask for image without tag (should be latest across tags, mongo)")
manifest = self.client.get_manifest(image="%s/%s" %(self.user_name,self.repo_name))
self.assertEqual(mongo,get_image_name(manifest))
print("Case 4: ask for latest tag (should be latest across tags, mongo)")
manifest = self.client.get_manifest(image="%s/%s:latest" %(self.user_name,self.repo_name))
self.assertEqual(mongo,get_image_name(manifest))
def test_get_image_name(self):
'''test_get_image_name will return the image name from the manifest
'''
from shub.api import get_image_name, get_manifest
manifest = get_manifest(image_id=self.image_id)
print("Case 1: return an image name using the commit id")
image_name = get_image_name(manifest)
self.assertEqual('f57e631a0434c31f0b4fa5276a314a6d8a672a55.img.gz',
image_name)
print("Case 2: ask for invalid extension")
with self.assertRaises(SystemExit) as cm:
image_name = get_image_name(manifest,
extension='.bz2')
self.assertEqual(cm.exception.code, 1)
print("Case 3: don't use commit (use md5 sum on generation)")
image_name = get_image_name(manifest,
use_commit=False)
print(image_name)
self.assertEqual('be4b9ba8fc22525d2ee2b27846513d42.img.gz',image_name)
def run(args):
# Find root filesystem location
if args.rootfs != None:
singularity_rootfs = args.rootfs
else:
singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
if singularity_rootfs == None and args.shub == None:
logger.error(
"root file system not specified OR defined as environmental variable, exiting!"
)
sys.exit(1)
if singularity_rootfs != None:
logger.info("Root file system defined as %s", singularity_rootfs)
# Does the registry require authentication?
auth = None
if args.username is not None and args.password is not None:
auth = basic_auth_header(args.username, args.password)
logger.info("Username for registry authentication: %s", args.username)
# Does the user want to download a Singularity image?
if args.shub != None:
image = args.shub
manifest = get_shub_manifest(image)
if args.pull_folder == None:
cache_base = get_cache(subfolder="shub",
disable_cache=args.disable_cache)
else:
cache_base = args.pull_folder
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
image_file = "%s/%s" % (cache_base, image_name)
if not os.path.exists(image_file):
image_file = download_image(manifest=manifest,
download_folder=cache_base)
else:
print("Image already exists at %s, skipping download." %
image_file)
logger.info("Singularity Hub Image Download: %s", image_file)
# If singularity_rootfs is provided, write metadata to it
if singularity_rootfs != None:
logger.debug(
"Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",
singularity_rootfs)
write_file("%s/SINGULARITY_RUNDIR" % singularity_rootfs,
os.path.dirname(image_file))
write_file("%s/SINGULARITY_IMAGE" % singularity_rootfs, image_file)
# Do we have a docker image specified?
elif args.docker != None:
# Does the user want to override default Entrypoint and use CMD as runscript?
includecmd = args.includecmd
logger.info("Including Docker command as Runscript? %s", includecmd)
image = args.docker
logger.info("Docker image: %s", image)
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
image = parse_image_uri(image=image, uri="docker://")
namespace = image['namespace']
repo_name = image['repo_name']
repo_tag = image['repo_tag']
# Tell the user the namespace, repo name and tag
logger.info("Docker image path: %s/%s:%s", namespace, repo_name,
repo_tag)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
manifest = get_manifest(repo_name=repo_name,
namespace=namespace,
repo_tag=repo_tag,
registry=args.registry,
auth=auth)
# Get images from manifest using version 2.0 of Docker Registry API
images = get_images(manifest=manifest)
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker",
disable_cache=args.disable_cache)
layers = []
for image_id in images:
# Download the layer, if we don't have it
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
targz = get_layer(image_id=image_id,
namespace=namespace,
repo_name=repo_name,
download_folder=cache_base,
registry=args.registry,
auth=auth)
layers.append(targz) # in case we want a list at the end
# Extract image and remove tar
output = extract_tar(targz, singularity_rootfs)
if output is None:
logger.error("Error extracting image: %s", targz)
sys.exit(1)
if args.disable_cache == True:
os.remove(targz)
# If the user wants to include the CMD as runscript, generate it here
if includecmd == True:
spec = "Cmd"
else:
spec = "Entrypoint"
cmd = get_config(manifest, spec=spec)
# Only add runscript if command is defined
if cmd != None:
print("Adding Docker %s as Singularity runscript..." %
(spec.upper()))
print(cmd)
runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)
# When we finish, clean up images
if args.disable_cache == True:
shutil.rmtree(cache_base)
logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def run(args):
# Find root filesystem location
if args.rootfs != None:
singularity_rootfs = args.rootfs
else:
singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
if singularity_rootfs == None and args.shub == None:
logger.error("root file system not specified OR defined as environmental variable, exiting!")
sys.exit(1)
if singularity_rootfs != None:
logger.info("Root file system defined as %s", singularity_rootfs)
# Does the registry require authentication?
auth = None
if args.username is not None and args.password is not None:
auth = basic_auth_header(args.username, args.password)
logger.info("Username for registry authentication: %s", args.username)
# Does the user want to download a Singularity image?
if args.shub != None:
image_id = int(args.shub)
manifest = get_shub_manifest(image_id)
cache_base = get_cache(subfolder="shub",
disable_cache = args.disable_cache)
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
image_file = "%s/%s" %(cache_base,image_name)
if not os.path.exists(image_file):
image_file = download_image(manifest=manifest,
download_folder=cache_base)
else:
print("Image already exists at %s, skipping download." %image_file)
logger.info("Singularity Hub Image Download: %s", image_file)
# If singularity_rootfs is provided, write metadata to it
if singularity_rootfs != None:
logger.debug("Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",singularity_rootfs)
write_file("%s/SINGULARITY_RUNDIR" %singularity_rootfs, os.path.dirname(image_file))
write_file("%s/SINGULARITY_IMAGE" %singularity_rootfs, image_file)
# Do we have a docker image specified?
elif args.docker != None:
# Does the user want to override default Entrypoint and use CMD as runscript?
includecmd = args.includecmd
logger.info("Including Docker command as Runscript? %s", includecmd)
image = args.docker
logger.info("Docker image: %s", image)
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
# First split the docker image name by /
image = image.split('/')
# If there are two parts, we have namespace with repo (and maybe tab)
if len(image) == 2:
namespace = image[0]
image = image[1]
# Otherwise, we must be using library namespace
else:
namespace = "library"
image = image[0]
# Now split the docker image name by :
image = image.split(':')
if len(image) == 2:
repo_name = image[0]
repo_tag = image[1]
# Otherwise, assume latest of an image
else:
repo_name = image[0]
repo_tag = "latest"
# Tell the user the namespace, repo name and tag
logger.info("Docker image path: %s/%s:%s", namespace,repo_name,repo_tag)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
manifest = get_manifest(repo_name=repo_name,
namespace=namespace,
repo_tag=repo_tag,
registry=args.registry,
auth=auth)
# Get images from manifest using version 2.0 of Docker Registry API
images = get_images(repo_name=repo_name,
namespace=namespace,
registry=args.registry,
auth=auth)
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker",
disable_cache = args.disable_cache)
layers = []
for image_id in images:
# Download the layer, if we don't have it
targz = "%s/%s.tar.gz" %(cache_base,image_id)
if not os.path.exists(targz):
targz = get_layer(image_id=image_id,
namespace=namespace,
repo_name=repo_name,
download_folder=cache_base,
registry=args.registry,
auth=auth)
layers.append(targz) # in case we want a list at the end
# Extract image and remove tar
output = extract_tar(targz,singularity_rootfs)
if output is None:
logger.error("Error extracting image: %s", targz)
sys.exit(1)
if args.disable_cache == True:
os.remove(targz)
# If the user wants to include the CMD as runscript, generate it here
if includecmd == True:
spec="Cmd"
else:
spec="Entrypoint"
cmd = get_config(manifest,spec=spec)
# Only add runscript if command is defined
if cmd != None:
print("Adding Docker %s as Singularity runscript..." %(spec.upper()))
print(cmd)
runscript = create_runscript(cmd=cmd,
base_dir=singularity_rootfs)
# When we finish, clean up images
if args.disable_cache == True:
shutil.rmtree(cache_base)
logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
