def copy_image_from_docker_archive(source_archive, dest_dir, remove_signatures=True): if remove_signatures: remove_signatures_string = "--remove-signatures" else: remove_signatures_string = "" cmdstr = "skopeo copy {} docker-archive:{} oci:{}:image".format( remove_signatures_string, source_archive, dest_dir ) cmd = cmdstr.split() try: rc, sout, serr = run_command_list(cmd) if rc != 0: raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) else: logger.debug( "command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip() ) except Exception as err: logger.error("command failed with exception - " + str(err)) raise err
def download_image(fulltag, copydir, user=None, pw=None, verify=True, manifest=None, use_cache_dir=None, dest_type='oci'): try: proc_env = os.environ.copy() if user and pw: proc_env['SKOPUSER'] = user proc_env['SKOPPASS'] = pw credstr = '--src-creds \"${SKOPUSER}\":\"${SKOPPASS}\"' else: credstr = "" if verify: tlsverifystr = "--src-tls-verify=true" else: tlsverifystr = "--src-tls-verify=false" if use_cache_dir and os.path.exists(use_cache_dir): cachestr = "--dest-shared-blob-dir " + use_cache_dir else: cachestr = "" if dest_type == 'oci': if manifest: with open(os.path.join(copydir, "manifest.json"), 'w') as OFH: OFH.write(manifest) cmd = [ "/bin/sh", "-c", "skopeo copy {} {} {} docker://{} oci:{}:image".format( tlsverifystr, credstr, cachestr, fulltag, copydir) ] else: cmd = [ "/bin/sh", "-c", "skopeo copy {} {} docker://{} dir:{}".format( tlsverifystr, credstr, fulltag, copydir) ] cmdstr = ' '.join(cmd) try: rc, sout, serr = run_command_list(cmd, env=proc_env) if rc != 0: raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) else: logger.debug("command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) except Exception as err: logger.error("command failed with exception - " + str(err)) raise err except Exception as err: raise err return (True)
def get_repo_tags_skopeo(url, registry, repo, user=None, pw=None, verify=None, lookuptag=None): try: proc_env = os.environ.copy() if user and pw: proc_env['SKOPUSER'] = user proc_env['SKOPPASS'] = pw credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"' else: credstr = "" if verify: tlsverifystr = "--tls-verify=true" else: tlsverifystr = "--tls-verify=false" localconfig = anchore_engine.configuration.localconfig.get_config() global_timeout = localconfig.get('skopeo_global_timeout', 0) try: global_timeout = int(global_timeout) if global_timeout < 0: global_timeout = 0 except: global_timeout = 0 if global_timeout: global_timeout_str = "--command-timeout {}s".format(global_timeout) else: global_timeout_str = "" pullstring = registry + "/" + repo if lookuptag: pullstring = pullstring + ":" + lookuptag repotags = [] cmd = ["/bin/sh", "-c", "skopeo {} inspect {} {} docker://{}".format(global_timeout_str, tlsverifystr, credstr, pullstring)] cmdstr = ' '.join(cmd) try: rc, sout, serr = run_command_list(cmd, env=proc_env) sout = str(sout, 'utf-8') if sout else None if rc != 0: raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) else: logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip()) except Exception as err: logger.error("command failed with exception - " + str(err)) raise err data = json.loads(sout) repotags = data.get('RepoTags', []) except Exception as err: raise err if not repotags: raise Exception("no tags found for input repo from skopeo") return(repotags)
def get_repo_tags_skopeo(url, registry, repo, user=None, pw=None, verify=None, lookuptag=None): try: proc_env = os.environ.copy() if user and pw: proc_env['SKOPUSER'] = user proc_env['SKOPPASS'] = pw credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"' else: credstr = "" if verify: tlsverifystr = "--tls-verify=true" else: tlsverifystr = "--tls-verify=false" pullstring = registry + "/" + repo if lookuptag: pullstring = pullstring + ":" + lookuptag repotags = [] cmd = [ "/bin/sh", "-c", "skopeo inspect {} {} docker://{}".format(tlsverifystr, credstr, pullstring) ] cmdstr = ' '.join(cmd) try: rc, sout, serr = run_command_list(cmd, env=proc_env) if rc != 0: raise Exception("command failed: cmd=" + str(cmdstr) + " exitcode=" + str(rc) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) else: logger.debug("command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) except Exception as err: logger.error("command failed with exception - " + str(err)) raise err data = json.loads(sout) repotags = data.get('RepoTags', []) except Exception as err: raise err if not repotags: raise Exception("no tags found for input repo from skopeo") return (repotags)
def get_image_manifest_skopeo_raw(pullstring, user=None, pw=None, verify=True): ret = None try: proc_env = os.environ.copy() if user and pw: proc_env['SKOPUSER'] = user proc_env['SKOPPASS'] = pw credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"' else: credstr = "" if verify: tlsverifystr = "--tls-verify=true" else: tlsverifystr = "--tls-verify=false" localconfig = anchore_engine.configuration.localconfig.get_config() global_timeout = localconfig.get('skopeo_global_timeout', 0) try: global_timeout = int(global_timeout) if global_timeout < 0: global_timeout = 0 except: global_timeout = 0 if global_timeout: global_timeout_str = "--command-timeout {}s".format(global_timeout) else: global_timeout_str = "" try: cmd = ["/bin/sh", "-c", "skopeo {} inspect --raw {} {} docker://{}".format(global_timeout_str, tlsverifystr, credstr, pullstring)] cmdstr = ' '.join(cmd) try: rc, sout, serr = run_command_list(cmd, env=proc_env) if rc != 0: raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) else: logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip()) except Exception as err: logger.error("command failed with exception - " + str(err)) raise err sout = str(sout, 'utf-8') if sout else None ret = sout except Exception as err: raise err except Exception as err: raise err return(ret)
def handle_tar_error(tarcmd, rc, sout, serr, unpackdir=None, rootfsdir=None, cachedir=None, layer=None, layertar=None, layers=[]): handled = False handled_post_metadata = { 'temporary_file_adds': [], 'temporary_dir_adds': [], } try: slinkre = "tar: (.*): Cannot open: File exists" hlinkre = "tar: (.*): Cannot hard link to .(.*).: No such file or directory" missingfiles = [] missingdirs = [] for errline in serr.splitlines(): patt = re.match(slinkre, errline) patt1 = re.match(hlinkre, errline) if patt: matchfile = patt.group(1) logger.debug("found 'file exists' error on name: " + str(matchfile)) if matchfile: badfile = os.path.join(rootfsdir, patt.group(1)) if os.path.exists(badfile): logger.debug("removing hierarchy: " + str(badfile)) shutil.rmtree(badfile) handled = True elif patt1: missingfile = patt1.group(2) basedir = os.path.dirname(missingfile) logger.debug( "found 'hard link' error on name: {}".format(missingfile)) if not os.path.exists(os.path.join(rootfsdir, missingfile)): missingfiles.append(missingfile) missingdir = None if not os.path.exists(os.path.join(rootfsdir, basedir)): missingdir = basedir missingdirs.append(missingdir) # only move on to further processing if the error is still not handled if not handled: if missingfiles: logger.info( "found {} missing hardlink destination files to extract from lower layers" .format(len(missingfiles))) for l in layers[layers.index("sha256:" + layer)::-1]: dighash, lname = l.split(":") ltar = get_layertarfile(unpackdir, cachedir, lname) tarcmd = "tar -C {} -x -f {}".format(rootfsdir, ltar) tarcmd_list = tarcmd.split() + missingfiles logger.debug( "attempting to run command to extract missing hardlink targets from layer {}: {}....." .format(l, tarcmd_list[:16])) rc, sout, serr = utils.run_command_list(tarcmd_list) sout = utils.ensure_str(sout) serr = utils.ensure_str(serr) #logger.debug("RESULT attempting to run command to extract missing hardlink target: {} : rc={} : serr={} : sout={}".format(tarcmd_list[:16], rc, serr, sout)) newmissingfiles = [] logger.debug( "missing file count before extraction at layer {}: {}". format(l, len(missingfiles))) for missingfile in missingfiles: tmpmissingfile = os.path.join(rootfsdir, missingfile) if os.path.exists(tmpmissingfile): if missingfile not in handled_post_metadata[ 'temporary_file_adds']: handled_post_metadata[ 'temporary_file_adds'].append(missingfile) else: if missingfile not in newmissingfiles: newmissingfiles.append(missingfile) missingfiles = newmissingfiles logger.debug( "missing file count after extraction at layer {}: {}". format(l, len(missingfiles))) newmissingdirs = [] for missingdir in missingdirs: tmpmissingdir = os.path.join(rootfsdir, missingdir) if os.path.exists(tmpmissingdir): if missingdir not in handled_post_metadata[ 'temporary_dir_adds']: handled_post_metadata[ 'temporary_dir_adds'].append(missingdir) else: if missingdir not in newmissingdirs: newmissingdirs.append(missingdir) missingdirs = newmissingdirs if not missingfiles: logger.info( "extraction of all missing files complete at layer {}" .format(l)) handled = True break else: logger.info( "extraction of all missing files not complete at layer {}, moving on to next layer" .format(l)) except Exception as err: raise err logger.debug("tar error handled: {}".format(handled)) return (handled, handled_post_metadata)
def download_image( fulltag, copydir, user=None, pw=None, verify=True, manifest=None, parent_manifest=None, use_cache_dir=None, remove_signatures=True, ): try: proc_env = os.environ.copy() if user and pw: proc_env["SKOPUSER"] = user proc_env["SKOPPASS"] = pw credstr = '--src-creds "${SKOPUSER}":"${SKOPPASS}"' else: credstr = "" if verify: tlsverifystr = "--src-tls-verify=true" else: tlsverifystr = "--src-tls-verify=false" if use_cache_dir and os.path.exists(use_cache_dir): cachestr = "--dest-shared-blob-dir " + use_cache_dir else: cachestr = "" localconfig = anchore_engine.configuration.localconfig.get_config() global_timeout = localconfig.get("skopeo_global_timeout", 0) try: global_timeout = int(global_timeout) if global_timeout < 0: global_timeout = 0 except: global_timeout = 0 if global_timeout: global_timeout_str = "--command-timeout {}s".format(global_timeout) else: global_timeout_str = "" os_overrides = [""] blobs_to_fetch = [] if manifest: manifest_data = json.loads(manifest) for layer in manifest_data.get("layers", []): if "foreign.diff" in layer.get("mediaType", ""): layer_digest_raw = layer.get("digest", "") layer_digest = get_digest_value(layer_digest_raw) layer_urls = layer.get("urls", []) blobs_to_fetch.append({ "digest": layer_digest, "urls": layer_urls }) if parent_manifest: parent_manifest_data = json.loads(parent_manifest) else: parent_manifest_data = {} if parent_manifest_data: for mlist in parent_manifest_data.get("manifests", []): imageos = mlist.get("platform", {}).get("os", "") if imageos not in ["", "linux"]: # add a windows os override to the list of override attempts, to complete the options that are supported by skopeo os_overrides.insert(0, "windows") break if remove_signatures: remove_signatures_string = "--remove-signatures" else: remove_signatures_string = "" for os_override in os_overrides: success = False if os_override not in ["", "linux"]: os_override_str = "--override-os {}".format(os_override) else: os_override_str = "" if manifest: with open(os.path.join(copydir, "manifest.json"), "w") as OFH: OFH.write(manifest) if parent_manifest: with open(os.path.join(copydir, "parent_manifest.json"), "w") as OFH: OFH.write(parent_manifest) cmd = [ "/bin/sh", "-c", "skopeo {} {} copy {} {} {} {} docker://{} oci:{}:image". format( os_override_str, global_timeout_str, remove_signatures_string, tlsverifystr, credstr, cachestr, fulltag, copydir, ), ] cmdstr = " ".join(cmd) try: rc, sout, serr = run_command_list(cmd, env=proc_env) if rc != 0: skopeo_error = SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) if skopeo_error.error_code != AnchoreError.OSARCH_MISMATCH.name: raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) else: logger.debug("command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) success = True except Exception as err: logger.error("command failed with exception - " + str(err)) raise err if success: blobs_dir = os.path.join(copydir, "blobs") if use_cache_dir: # syft expects blobs to be nested inside of the oci image directory. If the --dest-shared-blob-dir skopeo option is used we need to # provide access to the blobs via a symlink, as if the blobs were stored within the oci image directory if os.path.exists(blobs_dir) and os.path.isdir(blobs_dir): # if this directory is not empty, there is an issue and we should expect an exception os.rmdir(blobs_dir) os.symlink(use_cache_dir, blobs_dir) fetch_oci_blobs(blobs_dir, blobs_to_fetch) index_file_path = os.path.join(copydir, "index.json") ensure_no_nondistributable_media_types(index_file_path) ensure_layer_media_types_are_correct(copydir) break if not success: logger.error("could not download image") raise Exception("could not download image") except Exception as err: raise err return True
def get_image_manifest_skopeo_raw(pullstring, user=None, pw=None, verify=True): ret = None try: proc_env = os.environ.copy() if user and pw: proc_env["SKOPUSER"] = user proc_env["SKOPPASS"] = pw credstr = '--creds "${SKOPUSER}":"${SKOPPASS}"' else: credstr = "" if verify: tlsverifystr = "--tls-verify=true" else: tlsverifystr = "--tls-verify=false" localconfig = anchore_engine.configuration.localconfig.get_config() global_timeout = localconfig.get("skopeo_global_timeout", 0) try: global_timeout = int(global_timeout) if global_timeout < 0: global_timeout = 0 except: global_timeout = 0 if global_timeout: global_timeout_str = "--command-timeout {}s".format(global_timeout) else: global_timeout_str = "" os_override_strs = ["", "--override-os windows"] try: success = False for os_override_str in os_override_strs: cmd = [ "/bin/sh", "-c", "skopeo {} {} inspect --raw {} {} docker://{}".format( global_timeout_str, os_override_str, tlsverifystr, credstr, pullstring, ), ] cmdstr = " ".join(cmd) try: rc, sout, serr = run_command_list(cmd, env=proc_env) if rc != 0: skopeo_error = SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) if skopeo_error.error_code != AnchoreError.OSARCH_MISMATCH.name: raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) else: logger.debug("command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) success = True except Exception as err: logger.error("command failed with exception - " + str(err)) raise err if success: sout = str(sout, "utf-8") if sout else None ret = sout break if not success: logger.error("could not retrieve manifest") raise Exception("could not retrieve manifest") except Exception as err: raise err except Exception as err: raise err return ret
def get_image_manifest_skopeo(url, registry, repo, intag=None, indigest=None, user=None, pw=None, verify=True): manifest = {} digest = None testDigest = None if indigest: pullstring = registry + "/" + repo + "@" + indigest elif intag: pullstring = registry + "/" + repo + ":" + intag else: raise Exception("invalid input - must supply either an intag or indigest") try: proc_env = os.environ.copy() if user and pw: proc_env['SKOPUSER'] = user proc_env['SKOPPASS'] = pw credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"' else: credstr = "" if verify: tlsverifystr = "--tls-verify=true" else: tlsverifystr = "--tls-verify=false" try: cmd = ["/bin/sh", "-c", "skopeo inspect --raw {} {} docker://{}".format(tlsverifystr, credstr, pullstring)] cmdstr = ' '.join(cmd) try: rc, sout, serr = run_command_list(cmd, env=proc_env) if rc != 0: raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) else: logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip()) except Exception as err: logger.error("command failed with exception - " + str(err)) raise err sout = str(sout, 'utf-8') if sout else None digest = manifest_to_digest(sout) manifest = json.loads(sout) if manifest.get('schemaVersion') == 2 and manifest.get('mediaType') == 'application/vnd.docker.distribution.manifest.list.v2+json': # Get the arch-specific version for amd64 and linux new_digest = None for entry in manifest.get('manifests'): platform = entry.get('platform') if platform and platform.get('architecture') in ['amd64'] and platform.get('os') == 'linux': new_digest = entry.get('digest') break return get_image_manifest_skopeo(url=url, registry=registry, repo=repo, intag=None, indigest=new_digest, user=user, pw=pw, verify=verify) except Exception as err: logger.warn("CMD failed - exception: " + str(err)) raise err except Exception as err: import traceback traceback.print_exc() raise err if not manifest or not digest: raise SkopeoError(msg="No digest/manifest from skopeo") return(manifest, digest)
def download_image(fulltag, copydir, user=None, pw=None, verify=True, manifest=None, parent_manifest=None, use_cache_dir=None, dest_type='oci'): try: proc_env = os.environ.copy() if user and pw: proc_env['SKOPUSER'] = user proc_env['SKOPPASS'] = pw credstr = '--src-creds \"${SKOPUSER}\":\"${SKOPPASS}\"' else: credstr = "" if verify: tlsverifystr = "--src-tls-verify=true" else: tlsverifystr = "--src-tls-verify=false" if use_cache_dir and os.path.exists(use_cache_dir): cachestr = "--dest-shared-blob-dir " + use_cache_dir else: cachestr = "" localconfig = anchore_engine.configuration.localconfig.get_config() global_timeout = localconfig.get('skopeo_global_timeout', 0) try: global_timeout = int(global_timeout) if global_timeout < 0: global_timeout = 0 except: global_timeout = 0 if global_timeout: global_timeout_str = "--command-timeout {}s".format(global_timeout) else: global_timeout_str = "" os_overrides = [""] if manifest: manifest_data = json.loads(manifest) # skopeo doesn't support references in manifests for copy/download operations, with oci dest type - if found, override with dir dest_type for l in manifest_data.get('layers', []): if 'foreign.diff' in l.get('mediaType', ""): dest_type = 'dir' if parent_manifest: parent_manifest_data = json.loads(parent_manifest) else: parent_manifest_data = {} if parent_manifest_data: for mlist in parent_manifest_data.get('manifests', []): imageos = mlist.get('platform', {}).get('os', "") if imageos not in ["", 'linux'] and imageos not in os_overrides: dest_type = 'dir' os_overrides.insert(0, imageos) for os_override in os_overrides: success = False if os_override not in ["", 'linux']: dest_type = 'dir' os_override_str = "--override-os {}".format(os_override) else: os_override_str = "" if dest_type == 'oci': if manifest: with open(os.path.join(copydir, "manifest.json"), 'w') as OFH: OFH.write(manifest) if parent_manifest: with open(os.path.join(copydir, "parent_manifest.json"), 'w') as OFH: OFH.write(parent_manifest) cmd = ["/bin/sh", "-c", "skopeo {} {} copy {} {} {} docker://{} oci:{}:image".format(os_override_str, global_timeout_str, tlsverifystr, credstr, cachestr, fulltag, copydir)] else: cmd = ["/bin/sh", "-c", "skopeo {} {} copy {} {} docker://{} dir:{}".format(os_override_str, global_timeout_str, tlsverifystr, credstr, fulltag, copydir)] cmdstr = ' '.join(cmd) try: rc, sout, serr = run_command_list(cmd, env=proc_env) if rc != 0: skopeo_error = SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) if skopeo_error.error_code != AnchoreError.OSARCH_MISMATCH.name: raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) else: logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip()) success = True except Exception as err: logger.error("command failed with exception - " + str(err)) raise err if success: break if not success: logger.error("could not download image") raise Exception("could not download image") except Exception as err: raise err return(True)
def handle_tar_error(tarcmd, rc, sout, serr, unpackdir=None, rootfsdir=None, cachedir=None, layer=None, layertar=None, layers=[]): handled = False handled_post_metadata = {} try: slinkre = "tar: (.*): Cannot open: File exists" hlinkre = "tar: (.*): Cannot hard link to .(.*).: No such file or directory" for errline in serr.splitlines(): patt = re.match(slinkre, errline) patt1 = re.match(hlinkre, errline) if patt: matchfile = patt.group(1) logger.debug("found 'file exists' error on name: " + str(matchfile)) if matchfile: badfile = os.path.join(rootfsdir, patt.group(1)) if os.path.exists(badfile): logger.debug("removing hierarchy: " + str(badfile)) shutil.rmtree(badfile) handled = True elif patt1: missingfile = patt1.group(2) basedir = os.path.dirname(missingfile) logger.debug( "found 'hard link' error on name: {}".format(missingfile)) if not os.path.exists(os.path.join(rootfsdir, missingfile)): #for l in layers[layers.index("sha256:"+layer)::-1]: for l in layers[-1::-1]: missingdir = None if not os.path.exists(os.path.join(rootfsdir, basedir)): missingdir = basedir dighash, lname = l.split(":") ltar = get_layertarfile(unpackdir, cachedir, lname) tarcmd = "tar -C {} -x -f {}".format(rootfsdir, ltar) tarcmd_list = tarcmd.split() + [ "{}".format(missingfile) ] #logger.debug("attempting to run command to extract missing hardlink target from layer {}: {}".format(l, tarcmd_list)) rc, sout, serr = utils.run_command_list(tarcmd_list) sout = utils.ensure_str(sout) serr = utils.ensure_str(serr) #logger.debug("RESULT attempting to run command to extract missing hardlink target: {} : rc={} : serr={} : sout={}".format(tarcmd, rc, serr, sout)) if rc == 0: if not handled_post_metadata.get( 'temporary_file_adds', False): handled_post_metadata[ 'temporary_file_adds'] = [] handled_post_metadata[ 'temporary_file_adds'].append(missingfile) if missingdir: if not handled_post_metadata.get( 'temporary_dir_adds', False): handled_post_metadata[ 'temporary_dir_adds'] = [] handled_post_metadata[ 'temporary_dir_adds'].append(missingdir) handled = True break except Exception as err: raise err logger.debug("tar error handled: {}".format(handled)) return (handled, handled_post_metadata)