def get_tar_filenames(layertar):
ret = []
layertarfile = None
try:
logger.debug("using tarfile library to get file names")
layertarfile = tarfile.open(layertar,
mode='r',
format=tarfile.PAX_FORMAT)
ret = layertarfile.getnames()
except:
# python tarfile fils to unpack some docker image layers due to PAX header issue, try another method
logger.debug("using tar command to get file names")
tarcmd = "tar tf {}".format(layertar)
try:
ret = []
rc, sout, serr = utils.run_command(tarcmd)
if rc == 0 and sout:
for line in sout.splitlines():
re.sub("/+$", "", line)
ret.append(line)
else:
raise Exception("rc={} sout={} serr={}".format(rc, sout, serr))
except Exception as err:
logger.error("command failed with exception - " + str(err))
raise err
finally:
if layertarfile:
layertarfile.close()
return (ret)
def manifest_to_digest_shellout(rawmanifest):
ret = None
tmpmanifest = None
try:
fd, tmpmanifest = tempfile.mkstemp()
os.write(fd, rawmanifest)
os.close(fd)
cmd = "skopeo manifest-digest {}".format(tmpmanifest)
rc, sout, serr = run_command(cmd)
if rc == 0 and re.match("^sha256:.*", sout):
ret = sout.strip()
else:
logger.warn(
"failed to calculate digest from schema v1 manifest: cmd={} rc={} sout={} serr={}"
.format(cmd, rc, sout, serr))
raise Exception(
"failed to calculate digest from schema v1 manifest")
except Exception as err:
raise err
finally:
if tmpmanifest:
os.remove(tmpmanifest)
return (ret)
def run_anchore_analyzers(staging_dirs, imageDigest, imageId):
outputdir = staging_dirs['outputdir']
unpackdir = staging_dirs['unpackdir']
copydir = staging_dirs['copydir']
# run analyzers
anchore_module_root = resource_filename("anchore", "anchore-modules")
analyzer_root = os.path.join(anchore_module_root, "analyzers")
for f in os.listdir(analyzer_root):
thecmd = os.path.join(analyzer_root, f)
if re.match(".*\.py$", thecmd):
cmdstr = " ".join(
[thecmd, imageId, unpackdir, outputdir, unpackdir])
if True:
try:
rc, sout, serr = utils.run_command(cmdstr)
if rc != 0:
raise Exception("command failed: cmd=" + str(cmdstr) +
" exitcode=" + str(rc) + " stdout=" +
str(sout).strip() + " stderr=" +
str(serr).strip())
else:
logger.debug("command succeeded: cmd=" + str(cmdstr) +
" stdout=" + str(sout).strip() +
" stderr=" + str(serr).strip())
except Exception as err:
logger.error("command failed with exception - " + str(err))
#raise err
analyzer_manifest = {}
#TODO populate analyzer_manifest?
analyzer_report = {}
for analyzer_output in os.listdir(
os.path.join(outputdir, "analyzer_output")):
if analyzer_output not in analyzer_report:
analyzer_report[analyzer_output] = {}
for analyzer_output_el in os.listdir(
os.path.join(outputdir, "analyzer_output", analyzer_output)):
if analyzer_output_el not in analyzer_report[analyzer_output]:
analyzer_report[analyzer_output][analyzer_output_el] = {
'base': {}
}
data = read_kvfile_todict(
os.path.join(outputdir, "analyzer_output", analyzer_output,
analyzer_output_el))
if data:
analyzer_report[analyzer_output][analyzer_output_el][
'base'] = read_kvfile_todict(
os.path.join(outputdir, "analyzer_output",
analyzer_output, analyzer_output_el))
else:
analyzer_report[analyzer_output].pop(analyzer_output_el, None)
if not analyzer_report[analyzer_output]:
analyzer_report.pop(analyzer_output, None)
return (analyzer_report)
def handle_tar_error(tarcmd, rc, sout, serr, unpackdir=None, rootfsdir=None, layer=None, layertar=None, layers=[]):
handled = False
handled_post_metadata = {}
try:
slinkre = "tar: (.*): Cannot open: File exists"
hlinkre = "tar: (.*): Cannot hard link to `(.*)': No such file or directory"
for errline in serr.splitlines():
patt = re.match(slinkre, errline)
patt1 = re.match(hlinkre, errline)
if patt:
matchfile = patt.group(1)
logger.debug("found 'file exists' error on name: " + str(matchfile))
if matchfile:
badfile = os.path.join(rootfsdir, patt.group(1))
if os.path.exists(badfile):
logger.debug("removing hierarchy: " + str(badfile))
shutil.rmtree(badfile)
handled = True
elif patt1:
missingfile = patt1.group(2)
basedir = os.path.dirname(missingfile)
logger.debug("found 'hard link' error on name: {}".format(missingfile))
if not os.path.exists(os.path.join(rootfsdir, missingfile)):
for l in layers[layers.index("sha256:"+layer)::-1]:
missingdir = None
if not os.path.exists(os.path.join(rootfsdir, basedir)):
missingdir = basedir
tarcmd = "tar -C {} -x -f {} {}".format(rootfsdir, layertar, missingfile)
rc, sout, serr = utils.run_command(tarcmd)
sout = utils.ensure_str(sout)
serr = utils.ensure_str(serr)
if rc == 0:
if not handled_post_metadata.get('temporary_file_adds', False):
handled_post_metadata['temporary_file_adds'] = []
handled_post_metadata['temporary_file_adds'].append(missingfile)
if missingdir:
if not handled_post_metadata.get('temporary_dir_adds', False):
handled_post_metadata['temporary_dir_adds'] = []
handled_post_metadata['temporary_dir_adds'].append(missingdir)
handled = True
break
except Exception as err:
raise err
logger.debug("tar error handled: {}".format(handled))
return(handled, handled_post_metadata)
def manifest_to_digest_shellout(rawmanifest):
ret = None
tmpmanifest = None
try:
fd, tmpmanifest = tempfile.mkstemp()
os.write(fd, rawmanifest.encode("utf-8"))
os.close(fd)
localconfig = anchore_engine.configuration.localconfig.get_config()
global_timeout = localconfig.get("skopeo_global_timeout", 0)
try:
global_timeout = int(global_timeout)
if global_timeout < 0:
global_timeout = 0
except:
global_timeout = 0
if global_timeout:
global_timeout_str = "--command-timeout {}s".format(global_timeout)
else:
global_timeout_str = ""
cmd = "skopeo {} manifest-digest {}".format(global_timeout_str, tmpmanifest)
rc, sout, serr = run_command(cmd)
if rc == 0 and re.match("^sha256:.*", str(sout, "utf-8")):
ret = sout.strip()
else:
logger.warn(
"failed to calculate digest from schema v1 manifest: cmd={} rc={} sout={} serr={}".format(
cmd, rc, sout, serr
)
)
raise SkopeoError(
cmd=cmd,
rc=rc,
err=serr,
out=sout,
msg="Failed to calculate digest from schema v1 manifest",
)
except Exception as err:
raise err
finally:
if tmpmanifest:
os.remove(tmpmanifest)
return ret
def run_anchore_analyzers(staging_dirs, imageDigest, imageId, localconfig):
outputdir = staging_dirs['outputdir']
unpackdir = staging_dirs['unpackdir']
copydir = staging_dirs['copydir']
configdir = localconfig['service_dir']
# run analyzers
anchore_module_root = resource_filename("anchore_engine", "analyzers")
analyzer_root = os.path.join(anchore_module_root, "modules")
for f in list_analyzers():
cmdstr = " ".join([f, configdir, imageId, unpackdir, outputdir, unpackdir])
if True:
timer = time.time()
try:
rc, sout, serr = utils.run_command(cmdstr)
sout = utils.ensure_str(sout)
serr = utils.ensure_str(serr)
if rc != 0:
raise Exception("command failed: cmd="+str(cmdstr)+" exitcode="+str(rc)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip())
else:
logger.debug("command succeeded: cmd="+str(cmdstr)+" stdout="+str(sout).strip()+" stderr="+str(serr).strip())
except Exception as err:
logger.error("command failed with exception - " + str(err))
logger.debug("timing: specific analyzer time: {} - {}".format(f, time.time() - timer))
analyzer_report = {}
for analyzer_output in os.listdir(os.path.join(outputdir, "analyzer_output")):
if analyzer_output not in analyzer_report:
analyzer_report[analyzer_output] = {}
for analyzer_output_el in os.listdir(os.path.join(outputdir, "analyzer_output", analyzer_output)):
if analyzer_output_el not in analyzer_report[analyzer_output]:
analyzer_report[analyzer_output][analyzer_output_el] = {'base': {}}
data = read_kvfile_todict(os.path.join(outputdir, "analyzer_output", analyzer_output, analyzer_output_el))
if data:
analyzer_report[analyzer_output][analyzer_output_el]['base'] = read_kvfile_todict(os.path.join(outputdir, "analyzer_output", analyzer_output, analyzer_output_el))
else:
analyzer_report[analyzer_output].pop(analyzer_output_el, None)
if not analyzer_report[analyzer_output]:
analyzer_report.pop(analyzer_output, None)
return analyzer_report
def squash_orig(unpackdir, cachedir, layers):
rootfsdir = unpackdir + "/rootfs"
if os.path.exists(unpackdir + "/squashed.tar"):
return (True)
if not os.path.exists(rootfsdir):
os.makedirs(rootfsdir)
revlayer = list(layers)
revlayer.reverse()
l_excludes = {}
l_opqexcludes = {
} # stores list of special files to exclude only for next layer (.wh..wh..opq handling)
last_opqexcludes = {} # opq exlcudes for the last layer
for l in revlayer:
htype, layer = l.split(":", 1)
layertar = get_layertarfile(unpackdir, cachedir, layer)
count = 0
logger.debug("\tPass 1: " + str(layertar))
layertarfile = tarfile.open(layertar,
mode='r',
format=tarfile.PAX_FORMAT)
whpatt = re.compile(".*/\.wh\..*")
whopqpatt = re.compile(".*/\.wh\.\.wh\.\.opq")
l_opqexcludes[layer] = {}
myexcludes = {}
opqexcludes = {}
for member in layertarfile.getmembers():
# checks for whiteout conditions
if whopqpatt.match(member.name):
# found an opq entry, which means that this files in the next layer down (only) should not be included
fsub = re.sub(r"\.wh\.\.wh\.\.opq", "", member.name, 1)
# never include the whiteout file itself
myexcludes[member.name] = True
opqexcludes[fsub] = True
elif whpatt.match(member.name):
# found a normal whiteout, which means that this file in any lower layer should be excluded
fsub = re.sub(r"\.wh\.", "", member.name, 1)
# never include a whiteout file
myexcludes[member.name] = True
myexcludes[fsub] = True
else:
# if the last processed layer had an opq whiteout, check file to see if it lives in the opq directory
if last_opqexcludes:
dtoks = member.name.split("/")
for i in range(0, len(dtoks)):
dtok = '/'.join(dtoks[0:i])
dtokwtrail = '/'.join(dtoks[0:i]) + "/"
if dtok in last_opqexcludes or dtokwtrail in last_opqexcludes:
l_opqexcludes[layer][member.name] = True
break
# build up the list of excludes as we move down the layers
for l in l_excludes.keys():
myexcludes.update(l_excludes[l])
l_excludes[layer] = myexcludes
#last_opqexcludes = opqexcludes
last_opqexcludes.update(opqexcludes)
layertarfile.close()
logger.debug("Pass 3: untarring layers with exclusions")
imageSize = 0
for l in layers:
htype, layer = l.split(":", 1)
layertar = get_layertarfile(unpackdir, cachedir, layer)
imageSize = imageSize + os.path.getsize(layertar)
# write out the exluded files, adding the per-layer excludes if present
with open(unpackdir + "/efile", 'w') as OFH:
for efile in l_excludes[layer]:
OFH.write("%s\n" % efile)
if layer in l_opqexcludes and l_opqexcludes[layer]:
for efile in l_opqexcludes[layer]:
logger.debug("adding special for layer exclude: " +
str(efile))
OFH.write("%s\n" % efile)
retry = True
success = False
last_err = None
max_retries = 10
retries = 0
while (not success) and (retry):
tarcmd = "tar -C " + rootfsdir + " -x -X " + unpackdir + "/efile -f " + layertar
logger.debug("untarring squashed tarball: " + str(tarcmd))
try:
rc, sout, serr = utils.run_command(tarcmd)
if rc != 0:
logger.debug("tar error encountered, attempting to handle")
handled = handle_tar_error(tarcmd,
rc,
sout,
serr,
unpackdir=unpackdir,
rootfsdir=rootfsdir,
layer=layer,
layertar=layertar)
if not handled:
raise Exception("command failed: cmd=" + str(tarcmd) +
" exitcode=" + str(rc) + " stdout=" +
str(sout).strip() + " stderr=" +
str(serr).strip())
else:
logger.debug(
"tar error successfully handled, retrying")
else:
logger.debug("command succeeded: stdout=" +
str(sout).strip() + " stderr=" +
str(serr).strip())
success = True
except Exception as err:
logger.error("command failed with exception - " + str(err))
last_err = err
success = False
retry = False
# safety net
if retries > max_retries:
retry = False
retries = retries + 1
if not success:
if last_err:
raise last_err
else:
raise Exception("unknown exception in untar")
return ("done", imageSize)