def test_uri(self): '''test_uri will make sure that the endpoint returns the equivalent image for all different uri options ''' from shub.api import get_image_name, get_manifest manifest = get_manifest(image=self.image_id) image_name = get_image_name(manifest) print("Case 1: ask for image and ask for master branch (tag)") manifest = get_manifest(image="%s/%s:master" % (self.user_name, self.repo_name)) self.assertEqual(image_name, get_image_name(manifest)) print("Case 2: ask for different tag (mongo)") manifest = get_manifest(image="%s/%s:mongo" % (self.user_name, self.repo_name)) mongo = get_image_name(manifest) self.assertFalse(image_name == mongo) print( "Case 3: ask for image without tag (should be latest across tags, mongo)" ) manifest = get_manifest(image="%s/%s" % (self.user_name, self.repo_name)) self.assertEqual(mongo, get_image_name(manifest)) print( "Case 4: ask for latest tag (should be latest across tags, mongo)") manifest = get_manifest(image="%s/%s:latest" % (self.user_name, self.repo_name)) self.assertEqual(mongo, get_image_name(manifest))
def test_uri(self): '''test_uri will make sure that the endpoint returns the equivalent image for all different uri options ''' from shub.api import get_image_name manifest = self.client.get_manifest() image_name = get_image_name(manifest) fullname = "%s/%s" % (self.user_name, self.repo_name) print("Case 1: ask for image by tag.") client = self.connect(image="%s:latest" % fullname) manifest = client.get_manifest() image_name = get_image_name(manifest) self.assertEqual(image_name, get_image_name(manifest))
def test_uri(self): '''test_uri will make sure that the endpoint returns the equivalent image for all different uri options ''' print("Testing singularity hub uris") from shub.api import get_image_name manifest = self.client.get_manifest() image_name = get_image_name(manifest) fullname = "%s/%s" % (self.user_name, self.repo_name) print("...case 1: ask for image by tag.") client = self.connect(image="%s:latest" % fullname) manifest = client.get_manifest() image_name = get_image_name(manifest) self.assertEqual(image_name, get_image_name(manifest))
def test_get_image_name(self): '''test_get_image_name will return the image name from the manifest ''' from shub.api import get_image_name manifest = self.client.get_manifest() print("Case 1: return an image name corresponding to repo") image_name = get_image_name(manifest) self.assertEqual('vsoch-singularity-images-mongo.img.gz', image_name)
def test_get_image_name(self): '''test_get_image_name will return the image name from the manifest ''' print("Testing singularity hub image naming") from shub.api import get_image_name manifest = self.client.get_manifest() print("...case 1: return an image name corresponding to repo") image_name = get_image_name(manifest) self.assertEqual('vsoch-singularity-images-master-latest', image_name)
def test_get_image_name(self): '''test_get_image_name will return the image name from the manifest ''' from shub.api import get_image_name manifest = self.client.get_manifest(image=self.image_id) print("Case 1: return an image name corresponding to repo") image_name = get_image_name(manifest) self.assertEqual('vsoch-singularity-images-master.img.gz', image_name)
def test_get_image_name(self): '''test_get_image_name will return the image name from the manifest ''' from shub.api import get_image_name, get_manifest manifest = get_manifest(image_id=self.image_id) print("Case 1: return an image name using the commit id") image_name = get_image_name(manifest) self.assertEqual('f57e631a0434c31f0b4fa5276a314a6d8a672a55.img.gz', image_name) print("Case 2: ask for invalid extension") with self.assertRaises(SystemExit) as cm: image_name = get_image_name(manifest, extension='.bz2') self.assertEqual(cm.exception.code, 1) print("Case 3: don't use commit (use md5 sum on generation)") image_name = get_image_name(manifest, use_commit=False) print(image_name) self.assertEqual('be4b9ba8fc22525d2ee2b27846513d42.img.gz', image_name)
def test_get_image_name(self): '''test_get_image_name will return the image name from the manifest ''' from shub.api import get_image_name, get_manifest manifest = get_manifest(image=self.image_id) print("Case 1: return an image name using the commit id") image_name = get_image_name(manifest) self.assertEqual('6d3715a982865863ff20e8783014522edf1240e4.img.gz', image_name) print("Case 2: ask for invalid extension") with self.assertRaises(SystemExit) as cm: image_name = get_image_name(manifest, extension='.bz2') self.assertEqual(cm.exception.code, 1) print("Case 3: don't use commit (use md5 sum on generation)") image_name = get_image_name(manifest, use_commit=False) print(image_name) self.assertEqual('9e46ba8be1e10b1a2812844ac8072259.img.gz', image_name)
def test_uri(self): '''test_uri will make sure that the endpoint returns the equivalent image for all different uri options ''' from shub.api import get_image_name manifest = self.client.get_manifest() image_name = get_image_name(manifest) print("Case 1: ask for image and ask for master branch (tag)") manifest = self.client.get_manifest(image="%s/%s:master" %(self.user_name,self.repo_name)) image_name = get_image_name(manifest) self.assertEqual(image_name,get_image_name(manifest)) print("Case 2: ask for different tag (mongo)") manifest = self.client.get_manifest(image="%s/%s:mongo" %(self.user_name,self.repo_name)) mongo = get_image_name(manifest) self.assertFalse(image_name == mongo) print("Case 3: ask for image without tag (should be latest across tags, mongo)") manifest = self.client.get_manifest(image="%s/%s" %(self.user_name,self.repo_name)) self.assertEqual(mongo,get_image_name(manifest)) print("Case 4: ask for latest tag (should be latest across tags, mongo)") manifest = self.client.get_manifest(image="%s/%s:latest" %(self.user_name,self.repo_name)) self.assertEqual(mongo,get_image_name(manifest))
def test_get_image_name(self): '''test_get_image_name will return the image name from the manifest ''' from shub.api import get_image_name, get_manifest manifest = get_manifest(image_id=self.image_id) print("Case 1: return an image name using the commit id") image_name = get_image_name(manifest) self.assertEqual('f57e631a0434c31f0b4fa5276a314a6d8a672a55.img.gz', image_name) print("Case 2: ask for invalid extension") with self.assertRaises(SystemExit) as cm: image_name = get_image_name(manifest, extension='.bz2') self.assertEqual(cm.exception.code, 1) print("Case 3: don't use commit (use md5 sum on generation)") image_name = get_image_name(manifest, use_commit=False) print(image_name) self.assertEqual('be4b9ba8fc22525d2ee2b27846513d42.img.gz',image_name)
def run(args): # Find root filesystem location if args.rootfs != None: singularity_rootfs = args.rootfs else: singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None) if singularity_rootfs == None and args.shub == None: logger.error( "root file system not specified OR defined as environmental variable, exiting!" ) sys.exit(1) if singularity_rootfs != None: logger.info("Root file system defined as %s", singularity_rootfs) # Does the registry require authentication? auth = None if args.username is not None and args.password is not None: auth = basic_auth_header(args.username, args.password) logger.info("Username for registry authentication: %s", args.username) # Does the user want to download a Singularity image? if args.shub != None: image = args.shub manifest = get_shub_manifest(image) if args.pull_folder == None: cache_base = get_cache(subfolder="shub", disable_cache=args.disable_cache) else: cache_base = args.pull_folder # The image name is the md5 hash, download if it's not there image_name = get_image_name(manifest) image_file = "%s/%s" % (cache_base, image_name) if not os.path.exists(image_file): image_file = download_image(manifest=manifest, download_folder=cache_base) else: print("Image already exists at %s, skipping download." % image_file) logger.info("Singularity Hub Image Download: %s", image_file) # If singularity_rootfs is provided, write metadata to it if singularity_rootfs != None: logger.debug( "Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s", singularity_rootfs) write_file("%s/SINGULARITY_RUNDIR" % singularity_rootfs, os.path.dirname(image_file)) write_file("%s/SINGULARITY_IMAGE" % singularity_rootfs, image_file) # Do we have a docker image specified? elif args.docker != None: # Does the user want to override default Entrypoint and use CMD as runscript? includecmd = args.includecmd logger.info("Including Docker command as Runscript? %s", includecmd) image = args.docker logger.info("Docker image: %s", image) # Input Parsing ---------------------------- # Parse image name, repo name, and namespace image = parse_image_uri(image=image, uri="docker://") namespace = image['namespace'] repo_name = image['repo_name'] repo_tag = image['repo_tag'] # Tell the user the namespace, repo name and tag logger.info("Docker image path: %s/%s:%s", namespace, repo_name, repo_tag) # IMAGE METADATA ------------------------------------------- # Use Docker Registry API (version 2.0) to get images ids, manifest # Get an image manifest - has image ids to parse, and will be # used later to get Cmd manifest = get_manifest(repo_name=repo_name, namespace=namespace, repo_tag=repo_tag, registry=args.registry, auth=auth) # Get images from manifest using version 2.0 of Docker Registry API images = get_images(manifest=manifest) # DOWNLOAD LAYERS ------------------------------------------- # Each is a .tar.gz file, obtained from registry with curl # Get the cache (or temporary one) for docker cache_base = get_cache(subfolder="docker", disable_cache=args.disable_cache) layers = [] for image_id in images: # Download the layer, if we don't have it targz = "%s/%s.tar.gz" % (cache_base, image_id) if not os.path.exists(targz): targz = get_layer(image_id=image_id, namespace=namespace, repo_name=repo_name, download_folder=cache_base, registry=args.registry, auth=auth) layers.append(targz) # in case we want a list at the end # Extract image and remove tar output = extract_tar(targz, singularity_rootfs) if output is None: logger.error("Error extracting image: %s", targz) sys.exit(1) if args.disable_cache == True: os.remove(targz) # If the user wants to include the CMD as runscript, generate it here if includecmd == True: spec = "Cmd" else: spec = "Entrypoint" cmd = get_config(manifest, spec=spec) # Only add runscript if command is defined if cmd != None: print("Adding Docker %s as Singularity runscript..." % (spec.upper())) print(cmd) runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs) # When we finish, clean up images if args.disable_cache == True: shutil.rmtree(cache_base) logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def run(args): # Find root filesystem location if args.rootfs != None: singularity_rootfs = args.rootfs else: singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None) if singularity_rootfs == None and args.shub == None: logger.error("root file system not specified OR defined as environmental variable, exiting!") sys.exit(1) if singularity_rootfs != None: logger.info("Root file system defined as %s", singularity_rootfs) # Does the registry require authentication? auth = None if args.username is not None and args.password is not None: auth = basic_auth_header(args.username, args.password) logger.info("Username for registry authentication: %s", args.username) # Does the user want to download a Singularity image? if args.shub != None: image_id = int(args.shub) manifest = get_shub_manifest(image_id) cache_base = get_cache(subfolder="shub", disable_cache = args.disable_cache) # The image name is the md5 hash, download if it's not there image_name = get_image_name(manifest) image_file = "%s/%s" %(cache_base,image_name) if not os.path.exists(image_file): image_file = download_image(manifest=manifest, download_folder=cache_base) else: print("Image already exists at %s, skipping download." %image_file) logger.info("Singularity Hub Image Download: %s", image_file) # If singularity_rootfs is provided, write metadata to it if singularity_rootfs != None: logger.debug("Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",singularity_rootfs) write_file("%s/SINGULARITY_RUNDIR" %singularity_rootfs, os.path.dirname(image_file)) write_file("%s/SINGULARITY_IMAGE" %singularity_rootfs, image_file) # Do we have a docker image specified? elif args.docker != None: # Does the user want to override default Entrypoint and use CMD as runscript? includecmd = args.includecmd logger.info("Including Docker command as Runscript? %s", includecmd) image = args.docker logger.info("Docker image: %s", image) # Input Parsing ---------------------------- # Parse image name, repo name, and namespace # First split the docker image name by / image = image.split('/') # If there are two parts, we have namespace with repo (and maybe tab) if len(image) == 2: namespace = image[0] image = image[1] # Otherwise, we must be using library namespace else: namespace = "library" image = image[0] # Now split the docker image name by : image = image.split(':') if len(image) == 2: repo_name = image[0] repo_tag = image[1] # Otherwise, assume latest of an image else: repo_name = image[0] repo_tag = "latest" # Tell the user the namespace, repo name and tag logger.info("Docker image path: %s/%s:%s", namespace,repo_name,repo_tag) # IMAGE METADATA ------------------------------------------- # Use Docker Registry API (version 2.0) to get images ids, manifest # Get an image manifest - has image ids to parse, and will be # used later to get Cmd manifest = get_manifest(repo_name=repo_name, namespace=namespace, repo_tag=repo_tag, registry=args.registry, auth=auth) # Get images from manifest using version 2.0 of Docker Registry API images = get_images(repo_name=repo_name, namespace=namespace, registry=args.registry, auth=auth) # DOWNLOAD LAYERS ------------------------------------------- # Each is a .tar.gz file, obtained from registry with curl # Get the cache (or temporary one) for docker cache_base = get_cache(subfolder="docker", disable_cache = args.disable_cache) layers = [] for image_id in images: # Download the layer, if we don't have it targz = "%s/%s.tar.gz" %(cache_base,image_id) if not os.path.exists(targz): targz = get_layer(image_id=image_id, namespace=namespace, repo_name=repo_name, download_folder=cache_base, registry=args.registry, auth=auth) layers.append(targz) # in case we want a list at the end # Extract image and remove tar output = extract_tar(targz,singularity_rootfs) if output is None: logger.error("Error extracting image: %s", targz) sys.exit(1) if args.disable_cache == True: os.remove(targz) # If the user wants to include the CMD as runscript, generate it here if includecmd == True: spec="Cmd" else: spec="Entrypoint" cmd = get_config(manifest,spec=spec) # Only add runscript if command is defined if cmd != None: print("Adding Docker %s as Singularity runscript..." %(spec.upper())) print(cmd) runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs) # When we finish, clean up images if args.disable_cache == True: shutil.rmtree(cache_base) logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")