def load_image(self, image): self.image = parse_image_uri(image=image, uri='shub://', default_registry=SHUB_API_BASE, quiet=True) # parse_image_uri may return an empty namespace cause that's allowed # with docker://, but not with shub:// if len(self.image["namespace"]) == 0: bot.error("Namespace cannot be empty for shub:// url!") sys.exit(1)
def load_image(self, image): '''load_image parses the image uri, and loads the different image parameters into the client. The image should be a docker uri (eg docker://) or name of docker image. ''' image = parse_image_uri(image=image, uri="docker://") self.repo_name = image['repo_name'] self.repo_tag = image['repo_tag'] self.namespace = image['namespace'] self.version = image['version'] self.registry = image['registry'] self.update_token()
def load_image(self,image): '''load_image parses the image uri, and loads the different image parameters into the client. The image should be a docker uri (eg docker://) or name of docker image. ''' image = parse_image_uri(image=image,uri="docker://") self.repo_name = image['repo_name'] self.repo_tag = image['repo_tag'] self.namespace = image['namespace'] self.version = image['version'] self.registry = image['registry'] self.update_token()
def test_parse_image_uri(self): '''test_parse_image_uri ensures that the correct namespace, repo name, and tag (or unique id) is returned. ''' from shell import parse_image_uri print("Case 1: Specifying an shub:// image id should return a number") image = parse_image_uri(image="shub://7", uri="shub://") self.assertTrue(isinstance(image, int)) self.assertEqual(image, 7) print("Case 2: Checking for correct output tags in digest...") image_name = "%s/%s" % (self.namespace, self.repo_name) digest = parse_image_uri(image=image_name) for tag in ['repo_name', 'repo_tag', 'namespace']: self.assertTrue(tag in digest) print("Case 3: Tag when not specified should be latest.") self.assertTrue(digest['repo_tag'] == 'latest') print("Case 4: Tag when speciifed should be returned.") image_name = "%s/%s:%s" % (self.namespace, self.repo_name, "pusheenasaurus") digest = parse_image_uri(image=image_name) self.assertTrue(digest['repo_tag'] == 'pusheenasaurus') print("Case 5: Namespace when not specified should be library.") digest = parse_image_uri(image=self.repo_name) self.assertTrue(digest['repo_tag'] == 'latest') self.assertTrue(digest['namespace'] == 'library') print("Case 6: Repo name and tag without namespace...") image_name = "%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image=image_name) self.assertTrue(digest['repo_tag'] == self.tag) self.assertTrue(digest['namespace'] == 'library') self.assertTrue(digest['repo_name'] == self.repo_name) print("Case 7: Changing default namespace should not use library.") image_name = "%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image=image_name, default_namespace="meow") self.assertTrue(digest['namespace'] == 'meow')
def run(args): # Find root filesystem location if args.rootfs != None: singularity_rootfs = args.rootfs else: singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None) if singularity_rootfs == None and args.shub == None: logger.error( "root file system not specified OR defined as environmental variable, exiting!" ) sys.exit(1) if singularity_rootfs != None: logger.info("Root file system defined as %s", singularity_rootfs) # Does the registry require authentication? auth = None if args.username is not None and args.password is not None: auth = basic_auth_header(args.username, args.password) logger.info("Username for registry authentication: %s", args.username) # Does the user want to download a Singularity image? if args.shub != None: image = args.shub manifest = get_shub_manifest(image) if args.pull_folder == None: cache_base = get_cache(subfolder="shub", disable_cache=args.disable_cache) else: cache_base = args.pull_folder # The image name is the md5 hash, download if it's not there image_name = get_image_name(manifest) image_file = "%s/%s" % (cache_base, image_name) if not os.path.exists(image_file): image_file = download_image(manifest=manifest, download_folder=cache_base) else: print("Image already exists at %s, skipping download." % image_file) logger.info("Singularity Hub Image Download: %s", image_file) # If singularity_rootfs is provided, write metadata to it if singularity_rootfs != None: logger.debug( "Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s", singularity_rootfs) write_file("%s/SINGULARITY_RUNDIR" % singularity_rootfs, os.path.dirname(image_file)) write_file("%s/SINGULARITY_IMAGE" % singularity_rootfs, image_file) # Do we have a docker image specified? elif args.docker != None: # Does the user want to override default Entrypoint and use CMD as runscript? includecmd = args.includecmd logger.info("Including Docker command as Runscript? %s", includecmd) image = args.docker logger.info("Docker image: %s", image) # Input Parsing ---------------------------- # Parse image name, repo name, and namespace image = parse_image_uri(image=image, uri="docker://") namespace = image['namespace'] repo_name = image['repo_name'] repo_tag = image['repo_tag'] # Tell the user the namespace, repo name and tag logger.info("Docker image path: %s/%s:%s", namespace, repo_name, repo_tag) # IMAGE METADATA ------------------------------------------- # Use Docker Registry API (version 2.0) to get images ids, manifest # Get an image manifest - has image ids to parse, and will be # used later to get Cmd manifest = get_manifest(repo_name=repo_name, namespace=namespace, repo_tag=repo_tag, registry=args.registry, auth=auth) # Get images from manifest using version 2.0 of Docker Registry API images = get_images(manifest=manifest) # DOWNLOAD LAYERS ------------------------------------------- # Each is a .tar.gz file, obtained from registry with curl # Get the cache (or temporary one) for docker cache_base = get_cache(subfolder="docker", disable_cache=args.disable_cache) layers = [] for image_id in images: # Download the layer, if we don't have it targz = "%s/%s.tar.gz" % (cache_base, image_id) if not os.path.exists(targz): targz = get_layer(image_id=image_id, namespace=namespace, repo_name=repo_name, download_folder=cache_base, registry=args.registry, auth=auth) layers.append(targz) # in case we want a list at the end # Extract image and remove tar output = extract_tar(targz, singularity_rootfs) if output is None: logger.error("Error extracting image: %s", targz) sys.exit(1) if args.disable_cache == True: os.remove(targz) # If the user wants to include the CMD as runscript, generate it here if includecmd == True: spec = "Cmd" else: spec = "Entrypoint" cmd = get_config(manifest, spec=spec) # Only add runscript if command is defined if cmd != None: print("Adding Docker %s as Singularity runscript..." % (spec.upper())) print(cmd) runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs) # When we finish, clean up images if args.disable_cache == True: shutil.rmtree(cache_base) logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def test_parse_image_uri(self): '''test_parse_image_uri ensures that the correct namespace, repo name, and tag (or unique id) is returned. ''' from shell import parse_image_uri print("Case 1: Empty repo_name should return error") with self.assertRaises(SystemExit) as cm: image = parse_image_uri(image="") self.assertEqual(cm.exception.code, 1) print("Case 2: Checking for correct output tags in digest...") image_name = "%s/%s" % (self.namespace, self.repo_name) digest = parse_image_uri(image=image_name) for tag in ['registry', 'repo_name', 'repo_tag', 'namespace']: self.assertTrue(tag in digest) print("Case 3: Specifying only an image should return defaults") image = parse_image_uri(image="shub://lizardleezle", uri="shub://") self.assertTrue(isinstance(image, dict)) self.assertEqual(image["namespace"], self.NAMESPACE) self.assertEqual(image["repo_tag"], self.REPO_TAG) self.assertEqual(image["repo_name"], 'lizardleezle') self.assertEqual(image["registry"], self.REGISTRY) print("Case 4: Tag when speciifed should be returned.") image_name = "%s/%s:%s" % (self.namespace, self.repo_name, "pusheenasaurus") digest = parse_image_uri(image_name) self.assertTrue(digest['repo_tag'] == 'pusheenasaurus') print("Case 5: Repo name and tag without namespace...") image_name = "%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['repo_tag'] == self.tag) self.assertTrue(digest['namespace'] == 'library') self.assertTrue(digest['repo_name'] == self.repo_name) print("Case 6: Changing default namespace should not use library.") image_name = "meow/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['namespace'] == 'meow') print("Case 7: Changing default shouldn't use index.docker.io.") image_name = "meow/mix/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow') self.assertTrue(digest['namespace'] == 'mix') print("Case 8: Custom uri should use it.") image_name = "catdog://meow/mix/tenders/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="catdog://") self.assertTrue(digest['registry'] == 'meow') self.assertTrue(digest['namespace'] == 'mix/tenders') print("Case 9: Digest version should be parsed") image_name = ( "catdog://meow/mix/original/choice/%s:%s@sha:256xxxxxxxxxxxxxxx" # noqa % (self.repo_name, self.tag)) digest = parse_image_uri(image_name, uri="catdog://") self.assertTrue(digest['registry'] == 'meow') self.assertTrue(digest['namespace'] == 'mix/original/choice') self.assertTrue(digest['version'] == 'sha:256xxxxxxxxxxxxxxx')
def load_image(self, image): self.image = parse_image_uri(image=image, uri='shub://', default_registry=SHUB_API_BASE, quiet=True)
def test_parse_image_uri(self): '''test_parse_image_uri ensures that the correct namespace, repo name, and tag (or unique id) is returned. ''' from shell import parse_image_uri print("Case 1: Empty repo_name should return error") with self.assertRaises(SystemExit) as cm: image = parse_image_uri(image="") self.assertEqual(cm.exception.code, 1) print("Case 2: Checking for correct output tags in digest...") image_name = "%s/%s" % (self.namespace, self.repo_name) digest = parse_image_uri(image=image_name) for tag in ['registry', 'repo_name', 'repo_tag', 'namespace']: self.assertTrue(tag in digest) print("Case 3: Specifying only an image should return defaults") image = parse_image_uri(image="shub://lizardleezle", uri="shub://") self.assertTrue(isinstance(image, dict)) self.assertEqual(image["namespace"], self.NAMESPACE) self.assertEqual(image["repo_tag"], self.REPO_TAG) self.assertEqual(image["repo_name"], 'lizardleezle') self.assertEqual(image["registry"], self.REGISTRY) print("Case 4: Tag when specified should be returned.") image_name = "%s/%s:%s" % (self.namespace, self.repo_name, "pusheenasaurus") digest = parse_image_uri(image_name) self.assertTrue(digest['repo_tag'] == 'pusheenasaurus') print("Case 5: Repo name and tag without namespace...") image_name = "%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['repo_tag'] == self.tag) self.assertTrue(digest['namespace'] == self.NAMESPACE) self.assertTrue(digest['repo_name'] == self.repo_name) print("Case 6: Changing namespace should not use default.") image_name = "meow/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['namespace'] == 'meow') print("Case 7: Changing registry shouldn't use index.docker.io.") image_name = "meow/mix/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow') self.assertTrue(digest['namespace'] == 'mix') print("Case 8: Custom uri should use it.") image_name = "catdog://meow/mix/tenders/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="catdog://") self.assertTrue(digest['registry'] == 'meow') self.assertTrue(digest['namespace'] == 'mix/tenders') print("Case 9: Digest version should be parsed") image_name = ("catdog://meow/mix/original/choice/%s:%s@sha:256xxxxxxxxxxxxxxx" # noqa % (self.repo_name, self.tag)) digest = parse_image_uri(image_name, uri="catdog://") self.assertTrue(digest['registry'] == 'meow') self.assertTrue(digest['namespace'] == 'mix/original/choice') self.assertTrue(digest['version'] == 'sha:256xxxxxxxxxxxxxxx') # now test some tricky cases print("Case 10: registry and namespace, @version contains / and : (docker://)") image_name = "some.registry.com/mix/mux/repo@me/version-1:3:2" digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == 'some.registry.com') self.assertTrue(digest['repo_tag'] == self.REPO_TAG) self.assertTrue(digest['namespace'] == 'mix/mux') self.assertTrue(digest['repo_name'] == 'repo') self.assertTrue(digest['version'] == 'me/version-1:3:2') print("Case 11: registry and namespace, @version contains / and : (generic)") image_name = "registry/mix/mux/repo@me/version-1:3:2" digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'registry') self.assertTrue(digest['repo_tag'] == self.REPO_TAG) self.assertTrue(digest['namespace'] == 'mix/mux') self.assertTrue(digest['repo_name'] == 'repo') self.assertTrue(digest['version'] == 'me/version-1:3:2') print("Case 12: Namespaces can include / characters i.e. can be nested") image_name = "meow/mix/barf/baz/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow') self.assertTrue(digest['namespace'] == 'mix/barf/baz') print("Case 13: Namespaces can include '.'") image_name = "meow/mix.max/barf.baz/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow') self.assertTrue(digest['namespace'] == 'mix.max/barf.baz') self.assertTrue(digest['repo_tag'] == self.tag) self.assertTrue(digest['repo_name'] == self.repo_name) print("Case 14: registry contains ., default namespace") image_name = "meow.io/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow.io') self.assertTrue(digest['namespace'] == self.NAMESPACE) self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 15: registry contains :port, default namespace") # namespace is not allowed to be empty except for docker:// uris # so in this case, the default namespace is used image_name = "meow:123/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow:123') self.assertTrue(digest['namespace'] == self.NAMESPACE) self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 16: Namespace cannot be empty with full non-docker uri") image_name = "myuri://meow.io/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow.io') self.assertTrue(digest['namespace'] == self.NAMESPACE) self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 17: comments at the end of the line") image_name = "myuri://meow.io/mix/%s:%s # comment hel.lo/test:blah@stuff" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow.io') self.assertTrue(digest['namespace'] == 'mix') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 18: tag cannot contain @, version without : should be parsed") # apparently there was a bug where if @version doesn't contain any : character # it will get mis-parsed as part of the tag, which is clearly wrong image_name = "myuri://meow.io/mix/my-repo:tag-1.2.3@master" digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow.io') self.assertTrue(digest['namespace'] == 'mix') self.assertTrue(digest['repo_name'] == 'my-repo') self.assertTrue(digest['repo_tag'] == 'tag-1.2.3') self.assertTrue(digest['version'] == 'master') print("Case 19: tag cannot contain @, version without : should be parsed") # apparently there was a bug where if @version doesn't contain any : character # it will get mis-parsed as part of the tag, which is clearly wrong image_name = "myuri://meow.io/mix/my-repo:[email protected]" digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow.io') self.assertTrue(digest['namespace'] == 'mix') self.assertTrue(digest['repo_name'] == 'my-repo') self.assertTrue(digest['repo_tag'] == 'tag') self.assertTrue(digest['version'] == '2.2')
def test_parse_image_uri_docker(self): """ Docker-specific uri parsing rules """ from shell import parse_image_uri print("Case 1: just image, default everything else") digest = parse_image_uri(self.repo_name, uri="docker://") self.assertTrue(digest['registry'] == self.REGISTRY) self.assertTrue(digest['namespace'] == self.NAMESPACE) self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.REPO_TAG) print("Case 2: just image and tag, default everything else") image_name = "%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == self.REGISTRY) self.assertTrue(digest['namespace'] == self.NAMESPACE) self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 3: image, namespace and tag") image_name = "mix/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == self.REGISTRY) self.assertTrue(digest['namespace'] == 'mix') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 4: image, namespace, tag and version") image_name = "mix/%s:%s@sha:256xxxxxxxxxxxxxxx" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == self.REGISTRY) self.assertTrue(digest['namespace'] == 'mix') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) self.assertTrue(digest['version'] == 'sha:256xxxxxxxxxxxxxxx') print("Case 5: image, several namespaces and tag") # for docker, registry must have a . or a :port, else it's parsed # as a namespace. In this case, no registry is specified image_name = "mix/max/blitz/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == self.REGISTRY) self.assertTrue(digest['namespace'] == 'mix/max/blitz') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 6: image, namespace, tag and version") image_name = "mix/max/blitz/%s:%s@sha:256xxxxxxxxxxxxxxx" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == self.REGISTRY) self.assertTrue(digest['namespace'] == 'mix/max/blitz') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) self.assertTrue(digest['version'] == 'sha:256xxxxxxxxxxxxxxx') print("Case 7: registry with ., image and tag, empty namespace") # with docker://, if registry is present in uri, and namespace is empty, # we parse the namespace as empty image_name = "meow.io/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == 'meow.io') self.assertTrue(digest['namespace'] == '') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 8: registry with :port, image and tag, empty namespace") image_name = "meow:5000/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == 'meow:5000') self.assertTrue(digest['namespace'] == '') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 9: registry with . and :port, image and tag") image_name = "meow.io:5000/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == 'meow.io:5000') self.assertTrue(digest['namespace'] == '') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 10: registry with . and :port, image, no tag. empty namespace") image_name = "meow.io:5000/%s" % (self.repo_name) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == 'meow.io:5000') self.assertTrue(digest['namespace'] == '') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.REPO_TAG) print("Case 11: registry, image, namespace and tag") image_name = "meow.io/mix/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == 'meow.io') self.assertTrue(digest['namespace'] == 'mix') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 12: registry, image, several namespaces and tag") image_name = "meow:5000/mix/max/blitz/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == 'meow:5000') self.assertTrue(digest['namespace'] == 'mix/max/blitz') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 13: no registry, several namespaces containing . and tag") # a registry is matched if it contains : or ., but it must be the first one # before any other namespace, else it's just a namespace image_name = "mix/max.nix/blitz.krieg/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name, uri="docker://") self.assertTrue(digest['registry'] == self.REGISTRY) self.assertTrue(digest['namespace'] == 'mix/max.nix/blitz.krieg') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 15: full docker:// uri with registry") image_name = "docker://meow.io/mix/%s:%s" % (self.repo_name, self.tag) digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow.io') self.assertTrue(digest['namespace'] == 'mix') self.assertTrue(digest['repo_name'] == self.repo_name) self.assertTrue(digest['repo_tag'] == self.tag) print("Case 16: full docker uri with registry, empty namespace") image_name = "docker://*****:*****@, version without : should be parsed") # apparently there was a bug where if @version doesn't contain any : character # it will get mis-parsed as part of the tag, which is clearly wrong image_name = "docker://meow.io/mix/my-repo:tag-1.2.3@master" digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow.io') self.assertTrue(digest['namespace'] == 'mix') self.assertTrue(digest['repo_name'] == 'my-repo') self.assertTrue(digest['repo_tag'] == 'tag-1.2.3') self.assertTrue(digest['version'] == 'master') print("Case 19: tag cannot contain @, version without : should be parsed") # apparently there was a bug where if @version doesn't contain any : character # it will get mis-parsed as part of the tag, which is clearly wrong image_name = "docker://meow.io/mix/my-repo:[email protected]" digest = parse_image_uri(image_name) self.assertTrue(digest['registry'] == 'meow.io') self.assertTrue(digest['namespace'] == 'mix') self.assertTrue(digest['repo_name'] == 'my-repo') self.assertTrue(digest['repo_tag'] == 'tag') self.assertTrue(digest['version'] == '2.2')