示例#1
0
    def test_get_layer(self):
        '''test_get_layer will download docker layers
        '''
        from docker.api import get_layer
        from docker.api import get_images
        images = get_images(namespace=self.namespace, repo_name=self.repo_name)

        print("Case 1: Download an existing layer, should succeed")
        layer_file = get_layer(image_id=images[0],
                               repo_name=self.repo_name,
                               namespace=self.namespace,
                               download_folder=self.tmpdir)
        self.assertTrue(os.path.exists(layer_file))

        print("Case 2: Download a non existing layer, should fail")
        fake_layer = "sha256:111111111112222222222223333333333"
        with self.assertRaises(SystemExit) as cm:
            layer_file = get_layer(image_id=fake_layer,
                                   repo_name=self.repo_name,
                                   namespace=self.namespace,
                                   download_folder=self.tmpdir)
        self.assertEqual(cm.exception.code, 1)
示例#2
0
    def test_get_layer(self):
        '''test_get_layer will download docker layers
        '''
        from docker.api import get_layer
        from docker.api import get_images
        images = get_images(namespace = self.namespace,
                            repo_name = self.repo_name)
        
        print("Case 1: Download an existing layer, should succeed")
        layer_file = get_layer(image_id=images[0], 
                               repo_name = self.repo_name,
                               namespace = self.namespace,
                               download_folder = self.tmpdir)
        self.assertTrue(os.path.exists(layer_file))

        print("Case 2: Download a non existing layer, should fail")
        fake_layer = "sha256:111111111112222222222223333333333"
        with self.assertRaises(SystemExit) as cm:
            layer_file = get_layer(image_id=fake_layer, 
                                   repo_name = self.repo_name,
                                   namespace = self.namespace,
                                   download_folder = self.tmpdir)
        self.assertEqual(cm.exception.code, 1)
示例#3
0
def run(args):

    # Find root filesystem location
    if args.rootfs != None:
        singularity_rootfs = args.rootfs
    else:
        singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
        if singularity_rootfs == None and args.shub == None:
            logger.error(
                "root file system not specified OR defined as environmental variable, exiting!"
            )
            sys.exit(1)

    if singularity_rootfs != None:
        logger.info("Root file system defined as %s", singularity_rootfs)

    # Does the registry require authentication?
    auth = None
    if args.username is not None and args.password is not None:
        auth = basic_auth_header(args.username, args.password)
        logger.info("Username for registry authentication: %s", args.username)

    # Does the user want to download a Singularity image?
    if args.shub != None:
        image = args.shub
        manifest = get_shub_manifest(image)
        if args.pull_folder == None:
            cache_base = get_cache(subfolder="shub",
                                   disable_cache=args.disable_cache)
        else:
            cache_base = args.pull_folder

        # The image name is the md5 hash, download if it's not there
        image_name = get_image_name(manifest)
        image_file = "%s/%s" % (cache_base, image_name)
        if not os.path.exists(image_file):
            image_file = download_image(manifest=manifest,
                                        download_folder=cache_base)
        else:
            print("Image already exists at %s, skipping download." %
                  image_file)
        logger.info("Singularity Hub Image Download: %s", image_file)

        # If singularity_rootfs is provided, write metadata to it
        if singularity_rootfs != None:
            logger.debug(
                "Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",
                singularity_rootfs)
            write_file("%s/SINGULARITY_RUNDIR" % singularity_rootfs,
                       os.path.dirname(image_file))
            write_file("%s/SINGULARITY_IMAGE" % singularity_rootfs, image_file)

    # Do we have a docker image specified?
    elif args.docker != None:

        # Does the user want to override default Entrypoint and use CMD as runscript?
        includecmd = args.includecmd
        logger.info("Including Docker command as Runscript? %s", includecmd)

        image = args.docker
        logger.info("Docker image: %s", image)

        # Input Parsing ----------------------------
        # Parse image name, repo name, and namespace

        image = parse_image_uri(image=image, uri="docker://")
        namespace = image['namespace']
        repo_name = image['repo_name']
        repo_tag = image['repo_tag']

        # Tell the user the namespace, repo name and tag
        logger.info("Docker image path: %s/%s:%s", namespace, repo_name,
                    repo_tag)

        # IMAGE METADATA -------------------------------------------
        # Use Docker Registry API (version 2.0) to get images ids, manifest

        # Get an image manifest - has image ids to parse, and will be
        # used later to get Cmd
        manifest = get_manifest(repo_name=repo_name,
                                namespace=namespace,
                                repo_tag=repo_tag,
                                registry=args.registry,
                                auth=auth)

        # Get images from manifest using version 2.0 of Docker Registry API
        images = get_images(manifest=manifest)

        #  DOWNLOAD LAYERS -------------------------------------------
        # Each is a .tar.gz file, obtained from registry with curl

        # Get the cache (or temporary one) for docker
        cache_base = get_cache(subfolder="docker",
                               disable_cache=args.disable_cache)

        layers = []
        for image_id in images:

            # Download the layer, if we don't have it
            targz = "%s/%s.tar.gz" % (cache_base, image_id)

            if not os.path.exists(targz):
                targz = get_layer(image_id=image_id,
                                  namespace=namespace,
                                  repo_name=repo_name,
                                  download_folder=cache_base,
                                  registry=args.registry,
                                  auth=auth)

            layers.append(targz)  # in case we want a list at the end

            # Extract image and remove tar
            output = extract_tar(targz, singularity_rootfs)
            if output is None:
                logger.error("Error extracting image: %s", targz)
                sys.exit(1)
            if args.disable_cache == True:
                os.remove(targz)

        # If the user wants to include the CMD as runscript, generate it here
        if includecmd == True:
            spec = "Cmd"
        else:
            spec = "Entrypoint"

        cmd = get_config(manifest, spec=spec)

        # Only add runscript if command is defined
        if cmd != None:
            print("Adding Docker %s as Singularity runscript..." %
                  (spec.upper()))
            print(cmd)
            runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)

        # When we finish, clean up images
        if args.disable_cache == True:
            shutil.rmtree(cache_base)

        logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
示例#4
0
def run(args):

    # Find root filesystem location
    if args.rootfs != None:
       singularity_rootfs = args.rootfs
    else:
       singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
       if singularity_rootfs == None and args.shub == None: 
           logger.error("root file system not specified OR defined as environmental variable, exiting!")
           sys.exit(1)
    
    if singularity_rootfs != None:
        logger.info("Root file system defined as %s", singularity_rootfs)

    # Does the registry require authentication?
    auth = None
    if args.username is not None and args.password is not None:
        auth = basic_auth_header(args.username, args.password)
        logger.info("Username for registry authentication: %s", args.username)


    # Does the user want to download a Singularity image?
    if args.shub != None:
        image_id = int(args.shub)
        manifest = get_shub_manifest(image_id)

        cache_base = get_cache(subfolder="shub", 
                               disable_cache = args.disable_cache)

        # The image name is the md5 hash, download if it's not there
        image_name = get_image_name(manifest)
        image_file = "%s/%s" %(cache_base,image_name)
        if not os.path.exists(image_file):
            image_file = download_image(manifest=manifest,
                                        download_folder=cache_base)
        else:
            print("Image already exists at %s, skipping download." %image_file)
        logger.info("Singularity Hub Image Download: %s", image_file)
       
        # If singularity_rootfs is provided, write metadata to it
        if singularity_rootfs != None:
            logger.debug("Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",singularity_rootfs)
            write_file("%s/SINGULARITY_RUNDIR" %singularity_rootfs, os.path.dirname(image_file))
            write_file("%s/SINGULARITY_IMAGE" %singularity_rootfs, image_file)

    # Do we have a docker image specified?
    elif args.docker != None:

        # Does the user want to override default Entrypoint and use CMD as runscript?
        includecmd = args.includecmd
        logger.info("Including Docker command as Runscript? %s", includecmd)

        image = args.docker
        logger.info("Docker image: %s", image)

        # Input Parsing ----------------------------
        # Parse image name, repo name, and namespace

        # First split the docker image name by /
        image = image.split('/')

        # If there are two parts, we have namespace with repo (and maybe tab)
        if len(image) == 2:
            namespace = image[0]
            image = image[1]

        # Otherwise, we must be using library namespace
        else:
            namespace = "library"
            image = image[0]

        # Now split the docker image name by :
        image = image.split(':')
        if len(image) == 2:
            repo_name = image[0]
            repo_tag = image[1]

        # Otherwise, assume latest of an image
        else:
            repo_name = image[0]
            repo_tag = "latest"

        # Tell the user the namespace, repo name and tag
        logger.info("Docker image path: %s/%s:%s", namespace,repo_name,repo_tag)


        # IMAGE METADATA -------------------------------------------
        # Use Docker Registry API (version 2.0) to get images ids, manifest

        # Get an image manifest - has image ids to parse, and will be
        # used later to get Cmd
        manifest = get_manifest(repo_name=repo_name,
                                namespace=namespace,
                                repo_tag=repo_tag,
                                registry=args.registry,
                                auth=auth)

        # Get images from manifest using version 2.0 of Docker Registry API
        images = get_images(repo_name=repo_name,
                            namespace=namespace,
                            registry=args.registry,
                            auth=auth)
        
       
        #  DOWNLOAD LAYERS -------------------------------------------
        # Each is a .tar.gz file, obtained from registry with curl

        # Get the cache (or temporary one) for docker
        cache_base = get_cache(subfolder="docker", 
                               disable_cache = args.disable_cache)

        layers = []
        for image_id in images:

            # Download the layer, if we don't have it
            targz = "%s/%s.tar.gz" %(cache_base,image_id)
 
            if not os.path.exists(targz):
                targz = get_layer(image_id=image_id,
                                  namespace=namespace,
                                  repo_name=repo_name,
                                  download_folder=cache_base,
                                  registry=args.registry,
                                  auth=auth)

            layers.append(targz) # in case we want a list at the end

            # Extract image and remove tar
            output = extract_tar(targz,singularity_rootfs)
            if output is None:
                logger.error("Error extracting image: %s", targz)
                sys.exit(1)
            if args.disable_cache == True:
                os.remove(targz)
               
     
        # If the user wants to include the CMD as runscript, generate it here
        if includecmd == True:
            spec="Cmd"
        else:
            spec="Entrypoint"

        cmd = get_config(manifest,spec=spec)

        # Only add runscript if command is defined
        if cmd != None:
            print("Adding Docker %s as Singularity runscript..." %(spec.upper()))
            print(cmd)
            runscript = create_runscript(cmd=cmd,
                                         base_dir=singularity_rootfs)

        # When we finish, clean up images
        if args.disable_cache == True:
            shutil.rmtree(cache_base)


        logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
示例#5
0
def main():
    parser = argparse.ArgumentParser(
        description="bootstrap Docker images for Singularity containers")

    # Name of the docker image, required
    parser.add_argument(
        "--docker",
        dest='docker',
        help=
        "name of Docker image to bootstrap, in format library/ubuntu:latest",
        type=str,
        default=None)

    # root file system of singularity image
    parser.add_argument("--rootfs",
                        dest='rootfs',
                        help="the path for the root filesystem to extract to",
                        type=str,
                        default=None)

    # Docker registry (default is registry-1.docker.io
    parser.add_argument(
        "--registry",
        dest='registry',
        help="the registry path to use, to replace registry-1.docker.io",
        type=str,
        default=None)

    # Flag to add the Docker CMD as a runscript
    parser.add_argument(
        "--cmd",
        dest='includecmd',
        action="store_true",
        help=
        "boolean to specify that the CMD should be included as a runscript (default is not included)",
        default=False)

    # Flag to indicate a token is not required
    parser.add_argument(
        "--no-token",
        dest='notoken',
        action="store_true",
        help=
        "boolean to specify that the CMD should be included as a runscript (default is not included)",
        default=False)

    try:
        args = parser.parse_args()
    except:
        parser.print_help()
        sys.exit(0)

    # Find root filesystem location
    if args.rootfs != None:
        singularity_rootfs = args.rootfs
    else:
        singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
        if singularity_rootfs == None:
            print(
                "ERROR: root file system not specified or defined as environmental variable, exiting!"
            )
            sys.exit(1)

    # Does the registry require a token?
    doauth = True
    if args.notoken == True:
        doauth = False

    # Does the user want to include the CMD as runscript?
    includecmd = args.includecmd

    # Do we have a docker image specified?
    if args.docker != None:
        image = args.docker

        # INPUT PARSING -------------------------------------------
        # Parse image name, repo name, and namespace

        # First split the docker image name by /
        image = image.split('/')

        # If there are two parts, we have namespace with repo (and maybe tab)
        if len(image) == 2:
            namespace = image[0]
            image = image[1]

        # Otherwise, we must be using library namespace
        else:
            namespace = "library"
            image = image[0]

        # Now split the docker image name by :
        image = image.split(':')
        if len(image) == 2:
            repo_name = image[0]
            repo_tag = image[1]

        # Otherwise, assume latest of an image
        else:
            repo_name = image[0]
            repo_tag = "latest"

        # Tell the user the namespace, repo name and tag
        print("%s/%s:%s" % (namespace, repo_name, repo_tag))

        # IMAGE METADATA -------------------------------------------
        # Use Docker Registry API (version 2.0) to get images ids, manifest

        # Get an image manifest - has image ids to parse, and will be
        # used later to get Cmd
        manifest = get_manifest(repo_name=repo_name,
                                namespace=namespace,
                                repo_tag=repo_tag,
                                registry=args.registry,
                                auth=doauth)

        # Get images from manifest using version 2.0 of Docker Registry API
        images = get_images(manifest=manifest,
                            registry=args.registry,
                            auth=doauth)

        #  DOWNLOAD LAYERS -------------------------------------------
        # Each is a .tar.gz file, obtained from registry with curl

        # Create a temporary directory for targzs
        tmpdir = tempfile.mkdtemp()
        layers = []

        for image_id in images:

            # Download the layer
            targz = get_layer(image_id=image_id,
                              namespace=namespace,
                              repo_name=repo_name,
                              download_folder=tmpdir,
                              registry=args.registry,
                              auth=doauth)

            layers.append(targz)  # in case we want a list at the end
            # @chrisfilo suggestion to try compiling into one tar.gz

            # Extract image and remove tar
            extract_tar(targz, singularity_rootfs)
            os.remove(targz)

    # If the user wants to include the CMD as runscript, generate it here
    if includecmd == True:

        cmd = get_config(manifest)  # default is spec="Cmd"

        # Only add runscript if command is defined
        if cmd != None:
            print("Adding Docker CMD as Singularity runscript...")
            runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)

            # change permission of runscript to 0755 (default)
            change_permissions("%s/singularity" % (singularity_rootfs))
示例#6
0
def main():

    logger.info("\n*** STARTING DOCKER BOOTSTRAP PYTHON PORTION ****")

    parser = argparse.ArgumentParser(
        description="bootstrap Docker images for Singularity containers")

    # Name of the docker image, required
    parser.add_argument(
        "--docker",
        dest='docker',
        help=
        "name of Docker image to bootstrap, in format library/ubuntu:latest",
        type=str,
        default=None)

    # root file system of singularity image
    parser.add_argument("--rootfs",
                        dest='rootfs',
                        help="the path for the root filesystem to extract to",
                        type=str,
                        default=None)

    # Docker registry (default is registry-1.docker.io
    parser.add_argument(
        "--registry",
        dest='registry',
        help="the registry path to use, to replace registry-1.docker.io",
        type=str,
        default=None)

    # Flag to add the Docker CMD as a runscript
    parser.add_argument(
        "--cmd",
        dest='includecmd',
        action="store_true",
        help=
        "boolean to specify that CMD should be used instead of ENTRYPOINT as the runscript.",
        default=False)

    parser.add_argument("--username",
                        dest='username',
                        help="username for registry authentication",
                        default=None)

    parser.add_argument("--password",
                        dest='password',
                        help="password for registry authentication",
                        default=None)

    # Flag to disable cache
    parser.add_argument("--no-cache",
                        dest='disable_cache',
                        action="store_true",
                        help="boolean to specify disabling the cache.",
                        default=False)

    try:
        args = parser.parse_args()
    except:
        logger.error("Input args to %s improperly set, exiting.",
                     os.path.abspath(__file__))
        parser.print_help()
        sys.exit(0)

    # Find root filesystem location
    if args.rootfs != None:
        singularity_rootfs = args.rootfs
        logger.info("Root file system defined by command line variable as %s",
                    singularity_rootfs)
    else:
        singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
        if singularity_rootfs == None:
            logger.error(
                "root file system not specified OR defined as environmental variable, exiting!"
            )
            sys.exit(1)
        logger.info("Root file system defined by env variable as %s",
                    singularity_rootfs)

    # Does the registry require authentication?
    auth = None
    if args.username is not None and args.password is not None:
        auth = basic_auth_header(args.username, args.password)
        logger.info("Username for registry authentication: %s", args.username)

    # Does the user want to override default Entrypoint and use CMD as runscript?
    includecmd = args.includecmd
    logger.info("Including Docker command as Runscript? %s", includecmd)

    # Do we have a docker image specified?
    if args.docker != None:
        image = args.docker
        logger.info("Docker image: %s", image)

        # INPUT PARSING -------------------------------------------
        # Parse image name, repo name, and namespace

        # First split the docker image name by /
        image = image.split('/')

        # If there are two parts, we have namespace with repo (and maybe tab)
        if len(image) == 2:
            namespace = image[0]
            image = image[1]

        # Otherwise, we must be using library namespace
        else:
            namespace = "library"
            image = image[0]

        # Now split the docker image name by :
        image = image.split(':')
        if len(image) == 2:
            repo_name = image[0]
            repo_tag = image[1]

        # Otherwise, assume latest of an image
        else:
            repo_name = image[0]
            repo_tag = "latest"

        # Tell the user the namespace, repo name and tag
        logger.info("Docker image path: %s/%s:%s", namespace, repo_name,
                    repo_tag)

        # IMAGE METADATA -------------------------------------------
        # Use Docker Registry API (version 2.0) to get images ids, manifest

        # Get an image manifest - has image ids to parse, and will be
        # used later to get Cmd
        manifest = get_manifest(repo_name=repo_name,
                                namespace=namespace,
                                repo_tag=repo_tag,
                                registry=args.registry,
                                auth=auth)

        # Get images from manifest using version 2.0 of Docker Registry API
        images = get_images(manifest=manifest,
                            registry=args.registry,
                            auth=auth)

        #  DOWNLOAD LAYERS -------------------------------------------
        # Each is a .tar.gz file, obtained from registry with curl

        # Get the cache (or temporary one) for docker
        cache_base = get_cache(subfolder="docker",
                               disable_cache=args.disable_cache)

        layers = []
        for image_id in images:

            # Download the layer, if we don't have it
            targz = "%s/%s.tar.gz" % (cache_base, image_id)

            if not os.path.exists(targz):
                targz = get_layer(image_id=image_id,
                                  namespace=namespace,
                                  repo_name=repo_name,
                                  download_folder=cache_base,
                                  registry=args.registry,
                                  auth=auth)

            layers.append(targz)  # in case we want a list at the end
            # @chrisfilo suggestion to try compiling into one tar.gz

            # Extract image and remove tar
            extract_tar(targz, singularity_rootfs)
            if args.disable_cache == True:
                os.remove(targz)

        # If the user wants to include the CMD as runscript, generate it here
        if includecmd == True:
            spec = "Cmd"
        else:
            spec = "Entrypoint"

        cmd = get_config(manifest, spec=spec)

        # Only add runscript if command is defined
        if cmd != None:
            print("Adding Docker %s as Singularity runscript..." %
                  (spec.upper()))
            print(cmd)
            runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)

        # When we finish, clean up images
        if args.disable_cache == True:
            shutil.rmtree(cache_base)

        logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")