def copy_image_from_docker_archive(source_archive, dest_dir, remove_signatures=True):
    if remove_signatures:
        remove_signatures_string = "--remove-signatures"
    else:
        remove_signatures_string = ""

    cmdstr = "skopeo copy {} docker-archive:{} oci:{}:image".format(
        remove_signatures_string, source_archive, dest_dir
    )
    cmd = cmdstr.split()
    try:
        rc, sout, serr = run_command_list(cmd)
        if rc != 0:
            raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr)
        else:
            logger.debug(
                "command succeeded: cmd="
                + str(cmdstr)
                + " stdout="
                + str(sout).strip()
                + " stderr="
                + str(serr).strip()
            )

    except Exception as err:
        logger.error("command failed with exception - " + str(err))
        raise err
Example #2
0
def download_image(fulltag,
                   copydir,
                   user=None,
                   pw=None,
                   verify=True,
                   manifest=None,
                   use_cache_dir=None,
                   dest_type='oci'):
    try:
        proc_env = os.environ.copy()
        if user and pw:
            proc_env['SKOPUSER'] = user
            proc_env['SKOPPASS'] = pw
            credstr = '--src-creds \"${SKOPUSER}\":\"${SKOPPASS}\"'
        else:
            credstr = ""

        if verify:
            tlsverifystr = "--src-tls-verify=true"
        else:
            tlsverifystr = "--src-tls-verify=false"

        if use_cache_dir and os.path.exists(use_cache_dir):
            cachestr = "--dest-shared-blob-dir " + use_cache_dir
        else:
            cachestr = ""

        if dest_type == 'oci':
            if manifest:
                with open(os.path.join(copydir, "manifest.json"), 'w') as OFH:
                    OFH.write(manifest)
            cmd = [
                "/bin/sh", "-c",
                "skopeo copy {} {} {} docker://{} oci:{}:image".format(
                    tlsverifystr, credstr, cachestr, fulltag, copydir)
            ]
        else:
            cmd = [
                "/bin/sh", "-c", "skopeo copy {} {} docker://{} dir:{}".format(
                    tlsverifystr, credstr, fulltag, copydir)
            ]

        cmdstr = ' '.join(cmd)
        try:
            rc, sout, serr = run_command_list(cmd, env=proc_env)
            if rc != 0:
                raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr)
            else:
                logger.debug("command succeeded: cmd=" + str(cmdstr) +
                             " stdout=" + str(sout).strip() + " stderr=" +
                             str(serr).strip())

        except Exception as err:
            logger.error("command failed with exception - " + str(err))
            raise err
    except Exception as err:
        raise err

    return (True)
def get_repo_tags_skopeo(url, registry, repo, user=None, pw=None, verify=None, lookuptag=None):
    try:
        proc_env = os.environ.copy()
        if user and pw:
            proc_env['SKOPUSER'] = user
            proc_env['SKOPPASS'] = pw
            credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"'
        else:
            credstr = ""

        if verify:
            tlsverifystr = "--tls-verify=true"
        else:
            tlsverifystr = "--tls-verify=false"
            
        localconfig = anchore_engine.configuration.localconfig.get_config()
        global_timeout = localconfig.get('skopeo_global_timeout', 0)
        try:
            global_timeout = int(global_timeout)
            if global_timeout < 0:
                global_timeout = 0
        except:
            global_timeout = 0

        if global_timeout:
            global_timeout_str = "--command-timeout {}s".format(global_timeout)
        else:
            global_timeout_str = ""

        pullstring = registry + "/" + repo
        if lookuptag:
            pullstring = pullstring + ":" + lookuptag

        repotags = []

        cmd = ["/bin/sh", "-c", "skopeo {} inspect {} {} docker://{}".format(global_timeout_str, tlsverifystr, credstr, pullstring)]
        cmdstr = ' '.join(cmd)
        try:
            rc, sout, serr = run_command_list(cmd, env=proc_env)
            sout = str(sout, 'utf-8') if sout else None
            if rc != 0:
                raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr)
            else:
                logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip())
        except Exception as err:
            logger.error("command failed with exception - " + str(err))
            raise err

        data = json.loads(sout)
        repotags = data.get('RepoTags', [])
    except Exception as err:
        raise err

    if not repotags:
        raise Exception("no tags found for input repo from skopeo")

    return(repotags)
Example #4
0
def get_repo_tags_skopeo(url,
                         registry,
                         repo,
                         user=None,
                         pw=None,
                         verify=None,
                         lookuptag=None):
    try:
        proc_env = os.environ.copy()
        if user and pw:
            proc_env['SKOPUSER'] = user
            proc_env['SKOPPASS'] = pw
            credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"'
        else:
            credstr = ""

        if verify:
            tlsverifystr = "--tls-verify=true"
        else:
            tlsverifystr = "--tls-verify=false"

        pullstring = registry + "/" + repo
        if lookuptag:
            pullstring = pullstring + ":" + lookuptag

        repotags = []

        cmd = [
            "/bin/sh", "-c",
            "skopeo inspect {} {} docker://{}".format(tlsverifystr, credstr,
                                                      pullstring)
        ]
        cmdstr = ' '.join(cmd)
        try:
            rc, sout, serr = run_command_list(cmd, env=proc_env)
            if rc != 0:
                raise Exception("command failed: cmd=" + str(cmdstr) +
                                " exitcode=" + str(rc) + " stdout=" +
                                str(sout).strip() + " stderr=" +
                                str(serr).strip())
            else:
                logger.debug("command succeeded: cmd=" + str(cmdstr) +
                             " stdout=" + str(sout).strip() + " stderr=" +
                             str(serr).strip())
        except Exception as err:
            logger.error("command failed with exception - " + str(err))
            raise err

        data = json.loads(sout)
        repotags = data.get('RepoTags', [])
    except Exception as err:
        raise err

    if not repotags:
        raise Exception("no tags found for input repo from skopeo")

    return (repotags)
def get_image_manifest_skopeo_raw(pullstring, user=None, pw=None, verify=True):
    ret = None
    try:
        proc_env = os.environ.copy()
        if user and pw:
            proc_env['SKOPUSER'] = user
            proc_env['SKOPPASS'] = pw
            credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"'
        else:
            credstr = ""

        if verify:
            tlsverifystr = "--tls-verify=true"
        else:
            tlsverifystr = "--tls-verify=false"

        localconfig = anchore_engine.configuration.localconfig.get_config()
        global_timeout = localconfig.get('skopeo_global_timeout', 0)            
        try:
            global_timeout = int(global_timeout)
            if global_timeout < 0:
                global_timeout = 0
        except:
            global_timeout = 0

        if global_timeout:
            global_timeout_str = "--command-timeout {}s".format(global_timeout)
        else:
            global_timeout_str = ""

        try:
            cmd = ["/bin/sh", "-c", "skopeo {} inspect --raw {} {} docker://{}".format(global_timeout_str, tlsverifystr, credstr, pullstring)]
            cmdstr = ' '.join(cmd)
            try:
                rc, sout, serr = run_command_list(cmd, env=proc_env)
                if rc != 0:
                    raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr)
                else:
                    logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip())
            except Exception as err:
                logger.error("command failed with exception - " + str(err))
                raise err

            sout = str(sout, 'utf-8') if sout else None    
            ret = sout
        except Exception as err:
            raise err
    except Exception as err:
        raise err
    return(ret)
def handle_tar_error(tarcmd,
                     rc,
                     sout,
                     serr,
                     unpackdir=None,
                     rootfsdir=None,
                     cachedir=None,
                     layer=None,
                     layertar=None,
                     layers=[]):
    handled = False

    handled_post_metadata = {
        'temporary_file_adds': [],
        'temporary_dir_adds': [],
    }

    try:
        slinkre = "tar: (.*): Cannot open: File exists"
        hlinkre = "tar: (.*): Cannot hard link to .(.*).: No such file or directory"
        missingfiles = []
        missingdirs = []
        for errline in serr.splitlines():
            patt = re.match(slinkre, errline)
            patt1 = re.match(hlinkre, errline)
            if patt:
                matchfile = patt.group(1)
                logger.debug("found 'file exists' error on name: " +
                             str(matchfile))
                if matchfile:
                    badfile = os.path.join(rootfsdir, patt.group(1))
                    if os.path.exists(badfile):
                        logger.debug("removing hierarchy: " + str(badfile))
                        shutil.rmtree(badfile)
                        handled = True
            elif patt1:
                missingfile = patt1.group(2)
                basedir = os.path.dirname(missingfile)
                logger.debug(
                    "found 'hard link' error on name: {}".format(missingfile))
                if not os.path.exists(os.path.join(rootfsdir, missingfile)):
                    missingfiles.append(missingfile)

                missingdir = None
                if not os.path.exists(os.path.join(rootfsdir, basedir)):
                    missingdir = basedir
                    missingdirs.append(missingdir)

        # only move on to further processing if the error is still not handled
        if not handled:
            if missingfiles:
                logger.info(
                    "found {} missing hardlink destination files to extract from lower layers"
                    .format(len(missingfiles)))

                for l in layers[layers.index("sha256:" + layer)::-1]:
                    dighash, lname = l.split(":")
                    ltar = get_layertarfile(unpackdir, cachedir, lname)

                    tarcmd = "tar -C {} -x -f {}".format(rootfsdir, ltar)
                    tarcmd_list = tarcmd.split() + missingfiles
                    logger.debug(
                        "attempting to run command to extract missing hardlink targets from layer {}: {}....."
                        .format(l, tarcmd_list[:16]))

                    rc, sout, serr = utils.run_command_list(tarcmd_list)
                    sout = utils.ensure_str(sout)
                    serr = utils.ensure_str(serr)
                    #logger.debug("RESULT attempting to run command to extract missing hardlink target: {} : rc={} : serr={} : sout={}".format(tarcmd_list[:16], rc, serr, sout))

                    newmissingfiles = []
                    logger.debug(
                        "missing file count before extraction at layer {}: {}".
                        format(l, len(missingfiles)))
                    for missingfile in missingfiles:
                        tmpmissingfile = os.path.join(rootfsdir, missingfile)
                        if os.path.exists(tmpmissingfile):
                            if missingfile not in handled_post_metadata[
                                    'temporary_file_adds']:
                                handled_post_metadata[
                                    'temporary_file_adds'].append(missingfile)
                        else:
                            if missingfile not in newmissingfiles:
                                newmissingfiles.append(missingfile)
                    missingfiles = newmissingfiles
                    logger.debug(
                        "missing file count after extraction at layer {}: {}".
                        format(l, len(missingfiles)))

                    newmissingdirs = []
                    for missingdir in missingdirs:
                        tmpmissingdir = os.path.join(rootfsdir, missingdir)
                        if os.path.exists(tmpmissingdir):
                            if missingdir not in handled_post_metadata[
                                    'temporary_dir_adds']:
                                handled_post_metadata[
                                    'temporary_dir_adds'].append(missingdir)
                        else:
                            if missingdir not in newmissingdirs:
                                newmissingdirs.append(missingdir)
                    missingdirs = newmissingdirs

                    if not missingfiles:
                        logger.info(
                            "extraction of all missing files complete at layer {}"
                            .format(l))
                        handled = True
                        break
                    else:
                        logger.info(
                            "extraction of all missing files not complete at layer {}, moving on to next layer"
                            .format(l))

    except Exception as err:
        raise err

    logger.debug("tar error handled: {}".format(handled))
    return (handled, handled_post_metadata)
Example #7
0
def download_image(
    fulltag,
    copydir,
    user=None,
    pw=None,
    verify=True,
    manifest=None,
    parent_manifest=None,
    use_cache_dir=None,
    remove_signatures=True,
):
    try:
        proc_env = os.environ.copy()
        if user and pw:
            proc_env["SKOPUSER"] = user
            proc_env["SKOPPASS"] = pw
            credstr = '--src-creds "${SKOPUSER}":"${SKOPPASS}"'
        else:
            credstr = ""

        if verify:
            tlsverifystr = "--src-tls-verify=true"
        else:
            tlsverifystr = "--src-tls-verify=false"

        if use_cache_dir and os.path.exists(use_cache_dir):
            cachestr = "--dest-shared-blob-dir " + use_cache_dir
        else:
            cachestr = ""

        localconfig = anchore_engine.configuration.localconfig.get_config()
        global_timeout = localconfig.get("skopeo_global_timeout", 0)
        try:
            global_timeout = int(global_timeout)
            if global_timeout < 0:
                global_timeout = 0
        except:
            global_timeout = 0

        if global_timeout:
            global_timeout_str = "--command-timeout {}s".format(global_timeout)
        else:
            global_timeout_str = ""

        os_overrides = [""]
        blobs_to_fetch = []

        if manifest:
            manifest_data = json.loads(manifest)

            for layer in manifest_data.get("layers", []):
                if "foreign.diff" in layer.get("mediaType", ""):
                    layer_digest_raw = layer.get("digest", "")
                    layer_digest = get_digest_value(layer_digest_raw)
                    layer_urls = layer.get("urls", [])

                    blobs_to_fetch.append({
                        "digest": layer_digest,
                        "urls": layer_urls
                    })

            if parent_manifest:
                parent_manifest_data = json.loads(parent_manifest)
            else:
                parent_manifest_data = {}

            if parent_manifest_data:
                for mlist in parent_manifest_data.get("manifests", []):
                    imageos = mlist.get("platform", {}).get("os", "")
                    if imageos not in ["", "linux"]:
                        # add a windows os override to the list of override attempts, to complete the options that are supported by skopeo
                        os_overrides.insert(0, "windows")
                        break

        if remove_signatures:
            remove_signatures_string = "--remove-signatures"
        else:
            remove_signatures_string = ""

        for os_override in os_overrides:
            success = False
            if os_override not in ["", "linux"]:
                os_override_str = "--override-os {}".format(os_override)
            else:
                os_override_str = ""

            if manifest:
                with open(os.path.join(copydir, "manifest.json"), "w") as OFH:
                    OFH.write(manifest)

            if parent_manifest:
                with open(os.path.join(copydir, "parent_manifest.json"),
                          "w") as OFH:
                    OFH.write(parent_manifest)

            cmd = [
                "/bin/sh",
                "-c",
                "skopeo {} {} copy {} {} {} {} docker://{} oci:{}:image".
                format(
                    os_override_str,
                    global_timeout_str,
                    remove_signatures_string,
                    tlsverifystr,
                    credstr,
                    cachestr,
                    fulltag,
                    copydir,
                ),
            ]

            cmdstr = " ".join(cmd)
            try:
                rc, sout, serr = run_command_list(cmd, env=proc_env)
                if rc != 0:
                    skopeo_error = SkopeoError(cmd=cmd,
                                               rc=rc,
                                               out=sout,
                                               err=serr)
                    if skopeo_error.error_code != AnchoreError.OSARCH_MISMATCH.name:
                        raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr)
                else:
                    logger.debug("command succeeded: cmd=" + str(cmdstr) +
                                 " stdout=" + str(sout).strip() + " stderr=" +
                                 str(serr).strip())
                    success = True

            except Exception as err:
                logger.error("command failed with exception - " + str(err))
                raise err

            if success:
                blobs_dir = os.path.join(copydir, "blobs")

                if use_cache_dir:
                    # syft expects blobs to be nested inside of the oci image directory. If the --dest-shared-blob-dir skopeo option is used we need to
                    # provide access to the blobs via a symlink, as if the blobs were stored within the oci image directory
                    if os.path.exists(blobs_dir) and os.path.isdir(blobs_dir):
                        # if this directory is not empty, there is an issue and we should expect an exception
                        os.rmdir(blobs_dir)

                    os.symlink(use_cache_dir, blobs_dir)

                fetch_oci_blobs(blobs_dir, blobs_to_fetch)

                index_file_path = os.path.join(copydir, "index.json")
                ensure_no_nondistributable_media_types(index_file_path)
                ensure_layer_media_types_are_correct(copydir)

                break
        if not success:
            logger.error("could not download image")
            raise Exception("could not download image")
    except Exception as err:
        raise err

    return True
Example #8
0
def get_image_manifest_skopeo_raw(pullstring, user=None, pw=None, verify=True):
    ret = None
    try:
        proc_env = os.environ.copy()
        if user and pw:
            proc_env["SKOPUSER"] = user
            proc_env["SKOPPASS"] = pw
            credstr = '--creds "${SKOPUSER}":"${SKOPPASS}"'
        else:
            credstr = ""

        if verify:
            tlsverifystr = "--tls-verify=true"
        else:
            tlsverifystr = "--tls-verify=false"

        localconfig = anchore_engine.configuration.localconfig.get_config()
        global_timeout = localconfig.get("skopeo_global_timeout", 0)
        try:
            global_timeout = int(global_timeout)
            if global_timeout < 0:
                global_timeout = 0
        except:
            global_timeout = 0

        if global_timeout:
            global_timeout_str = "--command-timeout {}s".format(global_timeout)
        else:
            global_timeout_str = ""

        os_override_strs = ["", "--override-os windows"]
        try:
            success = False
            for os_override_str in os_override_strs:
                cmd = [
                    "/bin/sh",
                    "-c",
                    "skopeo {} {} inspect --raw {} {} docker://{}".format(
                        global_timeout_str,
                        os_override_str,
                        tlsverifystr,
                        credstr,
                        pullstring,
                    ),
                ]
                cmdstr = " ".join(cmd)
                try:
                    rc, sout, serr = run_command_list(cmd, env=proc_env)
                    if rc != 0:
                        skopeo_error = SkopeoError(cmd=cmd,
                                                   rc=rc,
                                                   out=sout,
                                                   err=serr)
                        if skopeo_error.error_code != AnchoreError.OSARCH_MISMATCH.name:
                            raise SkopeoError(cmd=cmd,
                                              rc=rc,
                                              out=sout,
                                              err=serr)
                    else:
                        logger.debug("command succeeded: cmd=" + str(cmdstr) +
                                     " stdout=" + str(sout).strip() +
                                     " stderr=" + str(serr).strip())
                        success = True
                except Exception as err:
                    logger.error("command failed with exception - " + str(err))
                    raise err

                if success:
                    sout = str(sout, "utf-8") if sout else None
                    ret = sout
                    break

            if not success:
                logger.error("could not retrieve manifest")
                raise Exception("could not retrieve manifest")

        except Exception as err:
            raise err
    except Exception as err:
        raise err

    return ret
def get_image_manifest_skopeo(url, registry, repo, intag=None, indigest=None, user=None, pw=None, verify=True):
    manifest = {}
    digest = None
    testDigest = None

    if indigest:
        pullstring = registry + "/" + repo + "@" + indigest
    elif intag:
        pullstring = registry + "/" + repo + ":" + intag
    else:
        raise Exception("invalid input - must supply either an intag or indigest")

    try:
        proc_env = os.environ.copy()
        if user and pw:
            proc_env['SKOPUSER'] = user
            proc_env['SKOPPASS'] = pw
            credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"'
        else:
            credstr = ""

        if verify:
            tlsverifystr = "--tls-verify=true"
        else:
            tlsverifystr = "--tls-verify=false"
            
        try:
            cmd = ["/bin/sh", "-c", "skopeo inspect --raw {} {} docker://{}".format(tlsverifystr, credstr, pullstring)]
            cmdstr = ' '.join(cmd)
            try:
                rc, sout, serr = run_command_list(cmd, env=proc_env)
                if rc != 0:
                    raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr)
                else:
                    logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip())
            except Exception as err:
                logger.error("command failed with exception - " + str(err))
                raise err

            sout = str(sout, 'utf-8') if sout else None
            digest = manifest_to_digest(sout)
            manifest = json.loads(sout)

            if manifest.get('schemaVersion') == 2 and manifest.get('mediaType') == 'application/vnd.docker.distribution.manifest.list.v2+json':
                # Get the arch-specific version for amd64 and linux
                new_digest = None
                for entry in manifest.get('manifests'):
                    platform = entry.get('platform')
                    if platform and platform.get('architecture') in ['amd64'] and platform.get('os') == 'linux':
                        new_digest = entry.get('digest')
                        break

                return get_image_manifest_skopeo(url=url, registry=registry, repo=repo, intag=None, indigest=new_digest, user=user, pw=pw, verify=verify)
        except Exception as err:
            logger.warn("CMD failed - exception: " + str(err))
            raise err

    except Exception as err:
        import traceback
        traceback.print_exc()
        raise err

    if not manifest or not digest:
        raise SkopeoError(msg="No digest/manifest from skopeo")

    return(manifest, digest)
def download_image(fulltag, copydir, user=None, pw=None, verify=True, manifest=None, parent_manifest=None, use_cache_dir=None, dest_type='oci'):
    try:
        proc_env = os.environ.copy()
        if user and pw:
            proc_env['SKOPUSER'] = user
            proc_env['SKOPPASS'] = pw
            credstr = '--src-creds \"${SKOPUSER}\":\"${SKOPPASS}\"'
        else:
            credstr = ""

        if verify:
            tlsverifystr = "--src-tls-verify=true"
        else:
            tlsverifystr = "--src-tls-verify=false"

        if use_cache_dir and os.path.exists(use_cache_dir):
            cachestr = "--dest-shared-blob-dir " + use_cache_dir
        else:
            cachestr = ""

        localconfig = anchore_engine.configuration.localconfig.get_config()
        global_timeout = localconfig.get('skopeo_global_timeout', 0)
        try:
            global_timeout = int(global_timeout)
            if global_timeout < 0:
                global_timeout = 0
        except:
            global_timeout = 0

        if global_timeout:
            global_timeout_str = "--command-timeout {}s".format(global_timeout)
        else:
            global_timeout_str = ""

        os_overrides = [""]
        if manifest:
            manifest_data = json.loads(manifest)

            # skopeo doesn't support references in manifests for copy/download operations, with oci dest type - if found, override with dir dest_type
            for l in manifest_data.get('layers', []):
                if 'foreign.diff' in l.get('mediaType', ""):
                    dest_type = 'dir'

            if parent_manifest:
                parent_manifest_data = json.loads(parent_manifest)
            else:
                parent_manifest_data = {}

            if parent_manifest_data:
                for mlist in parent_manifest_data.get('manifests', []):
                    imageos = mlist.get('platform', {}).get('os', "")
                    if imageos not in ["", 'linux'] and imageos not in os_overrides:
                        dest_type = 'dir'
                        os_overrides.insert(0, imageos)

        for os_override in os_overrides:
            success = False
            if os_override not in ["", 'linux']:
                dest_type = 'dir'
                os_override_str = "--override-os {}".format(os_override)
            else:
                os_override_str = ""
                
            if dest_type == 'oci':
                if manifest:
                    with open(os.path.join(copydir, "manifest.json"), 'w') as OFH:
                        OFH.write(manifest)

                if parent_manifest:
                    with open(os.path.join(copydir, "parent_manifest.json"), 'w') as OFH:
                        OFH.write(parent_manifest)
                        
                cmd = ["/bin/sh", "-c", "skopeo {} {} copy {} {} {} docker://{} oci:{}:image".format(os_override_str, global_timeout_str, tlsverifystr, credstr, cachestr, fulltag, copydir)]
            else:
                cmd = ["/bin/sh", "-c", "skopeo {} {} copy {} {} docker://{} dir:{}".format(os_override_str, global_timeout_str, tlsverifystr, credstr, fulltag, copydir)]

            cmdstr = ' '.join(cmd)
            try:
                rc, sout, serr = run_command_list(cmd, env=proc_env)
                if rc != 0:
                    skopeo_error = SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr)
                    if skopeo_error.error_code != AnchoreError.OSARCH_MISMATCH.name:
                        raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr)                    
                else:
                    logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip())
                    success = True                    

            except Exception as err:
                logger.error("command failed with exception - " + str(err))
                raise err

            if success:
                break
        if not success:
            logger.error("could not download image")
            raise Exception("could not download image")
    except Exception as err:
        raise err

    return(True)
def handle_tar_error(tarcmd,
                     rc,
                     sout,
                     serr,
                     unpackdir=None,
                     rootfsdir=None,
                     cachedir=None,
                     layer=None,
                     layertar=None,
                     layers=[]):
    handled = False
    handled_post_metadata = {}

    try:
        slinkre = "tar: (.*): Cannot open: File exists"
        hlinkre = "tar: (.*): Cannot hard link to .(.*).: No such file or directory"

        for errline in serr.splitlines():
            patt = re.match(slinkre, errline)
            patt1 = re.match(hlinkre, errline)
            if patt:
                matchfile = patt.group(1)
                logger.debug("found 'file exists' error on name: " +
                             str(matchfile))
                if matchfile:
                    badfile = os.path.join(rootfsdir, patt.group(1))
                    if os.path.exists(badfile):
                        logger.debug("removing hierarchy: " + str(badfile))
                        shutil.rmtree(badfile)
                        handled = True
            elif patt1:
                missingfile = patt1.group(2)
                basedir = os.path.dirname(missingfile)
                logger.debug(
                    "found 'hard link' error on name: {}".format(missingfile))
                if not os.path.exists(os.path.join(rootfsdir, missingfile)):
                    #for l in layers[layers.index("sha256:"+layer)::-1]:
                    for l in layers[-1::-1]:

                        missingdir = None
                        if not os.path.exists(os.path.join(rootfsdir,
                                                           basedir)):
                            missingdir = basedir

                        dighash, lname = l.split(":")
                        ltar = get_layertarfile(unpackdir, cachedir, lname)

                        tarcmd = "tar -C {} -x -f {}".format(rootfsdir, ltar)
                        tarcmd_list = tarcmd.split() + [
                            "{}".format(missingfile)
                        ]
                        #logger.debug("attempting to run command to extract missing hardlink target from layer {}: {}".format(l, tarcmd_list))
                        rc, sout, serr = utils.run_command_list(tarcmd_list)
                        sout = utils.ensure_str(sout)
                        serr = utils.ensure_str(serr)
                        #logger.debug("RESULT attempting to run command to extract missing hardlink target: {} : rc={} : serr={} : sout={}".format(tarcmd, rc, serr, sout))
                        if rc == 0:
                            if not handled_post_metadata.get(
                                    'temporary_file_adds', False):
                                handled_post_metadata[
                                    'temporary_file_adds'] = []
                            handled_post_metadata[
                                'temporary_file_adds'].append(missingfile)

                            if missingdir:
                                if not handled_post_metadata.get(
                                        'temporary_dir_adds', False):
                                    handled_post_metadata[
                                        'temporary_dir_adds'] = []
                                handled_post_metadata[
                                    'temporary_dir_adds'].append(missingdir)

                            handled = True
                            break

    except Exception as err:
        raise err

    logger.debug("tar error handled: {}".format(handled))
    return (handled, handled_post_metadata)