def run_anchore_analyzers(staging_dirs, imageDigest, imageId):
outputdir = staging_dirs['outputdir']
unpackdir = staging_dirs['unpackdir']
copydir = staging_dirs['copydir']
# run analyzers
anchore_module_root = resource_filename("anchore", "anchore-modules")
analyzer_root = os.path.join(anchore_module_root, "analyzers")
for f in os.listdir(analyzer_root):
thecmd = os.path.join(analyzer_root, f)
if re.match(".*\.py$", thecmd):
cmdstr = " ".join(
[thecmd, imageId, unpackdir, outputdir, unpackdir])
if True:
try:
rc, sout, serr = anchore_engine.services.common.run_command(
cmdstr)
if rc != 0:
raise Exception("command failed: cmd=" + str(cmdstr) +
" exitcode=" + str(rc) + " stdout=" +
str(sout).strip() + " stderr=" +
str(serr).strip())
else:
logger.debug("command succeeded: cmd=" + str(cmdstr) +
" stdout=" + str(sout).strip() +
" stderr=" + str(serr).strip())
except Exception as err:
logger.error("command failed with exception - " + str(err))
#raise err
analyzer_manifest = {}
#TODO populate analyzer_manifest?
analyzer_report = {}
for analyzer_output in os.listdir(
os.path.join(outputdir, "analyzer_output")):
if analyzer_output not in analyzer_report:
analyzer_report[analyzer_output] = {}
for analyzer_output_el in os.listdir(
os.path.join(outputdir, "analyzer_output", analyzer_output)):
if analyzer_output_el not in analyzer_report[analyzer_output]:
analyzer_report[analyzer_output][analyzer_output_el] = {
'base': {}
}
data = read_kvfile_todict(
os.path.join(outputdir, "analyzer_output", analyzer_output,
analyzer_output_el))
if data:
analyzer_report[analyzer_output][analyzer_output_el][
'base'] = read_kvfile_todict(
os.path.join(outputdir, "analyzer_output",
analyzer_output, analyzer_output_el))
else:
analyzer_report[analyzer_output].pop(analyzer_output_el, None)
if not analyzer_report[analyzer_output]:
analyzer_report.pop(analyzer_output, None)
return (analyzer_report)
def load_analysis_output(self, imageId, module_name, module_value, module_type=None):
ret = {}
if not module_type or module_type == 'base':
thefile = '/'.join([self.imagerootdir, imageId, "analyzer_output", module_name, module_value])
else:
thefile = '/'.join([self.imagerootdir, imageId, "analyzer_output_"+module_type, module_name, module_value])
if os.path.exists(thefile):
if os.path.isfile(thefile):
ret = anchore_utils.read_kvfile_todict(thefile)
elif os.path.isdir(thefile):
ret = thefile
import tarfile, io
try:
TFH = io.BytesIO()
tar = tarfile.open(fileobj=TFH, mode='w:gz', format=tarfile.PAX_FORMAT)
for l in os.listdir(thefile):
tarfile = os.path.join(thefile, l)
tar.add(tarfile)
tar.close()
TFH.seek(0)
ret = TFH
except Exception as err:
import traceback
traceback.print_exc()
print str(err)
return(ret)
def load_analysis_output(self, imageId, module_name, module_value, module_type=None):
ret = {}
if not module_type or module_type == 'base':
thefile = '/'.join([self.imagerootdir, imageId, "analyzer_output", module_name, module_value])
else:
thefile = '/'.join([self.imagerootdir, imageId, "analyzer_output_"+module_type, module_name, module_value])
if os.path.exists(thefile):
if os.path.isfile(thefile):
ret = anchore_utils.read_kvfile_todict(thefile)
elif os.path.isdir(thefile):
ret = thefile
import tarfile, io
try:
TFH = io.BytesIO()
tar = tarfile.open(fileobj=TFH, mode='w:gz', format=tarfile.PAX_FORMAT)
for l in os.listdir(thefile):
tarfile = os.path.join(thefile, l)
tar.add(tarfile)
tar.close()
TFH.seek(0)
ret = TFH
except Exception as err:
import traceback
traceback.print_exc()
print str(err)
return(ret)
def run_analyzers(self, image):
success = True
analyzers = self.list_analyzers()
imagename = image.meta['imagename']
outputdir = image.anchore_imagedir
shortid = image.meta['shortId']
imagedir = None
analyzer_status = self.anchoreDB.load_analyzer_manifest(
image.meta['imageId'])
results = {}
outputdirs = {}
torun = list()
skip = False
for atype in ['user_scripts_dir', 'extra_scripts_dir', 'base']:
for script in analyzers[atype]:
try:
with open(script, 'r') as FH:
csum = hashlib.md5(FH.read()).hexdigest()
except:
csum = "N/A"
# decide whether or not to run the analyzer
dorun = True
if self.force:
dorun = True
elif script in analyzer_status:
if csum == analyzer_status[script][
'csum'] and analyzer_status[script][
'returncode'] == 0:
dorun = False
outputdir = cmdstr = outstr = ""
if dorun:
if not skip:
if not imagedir:
self._logger.info(image.meta['shortId'] +
": analyzing ...")
imagedir = image.unpack()
outputdir = tempfile.mkdtemp(dir=imagedir)
cmdline = ' '.join([
imagename, self.config['image_data_store'],
outputdir, imagedir
])
cmdstr = script + " " + cmdline
cmd = cmdstr.split()
try:
self._logger.debug("running analyzer: " + cmdstr)
timer = time.time()
outstr = subprocess.check_output(
cmd, stderr=subprocess.STDOUT)
self._logger.debug("analyzer time (seconds): " +
str(time.time() - timer))
rc = 0
self._logger.debug("analyzer status: success")
self._logger.debug("analyzer exitcode: " + str(rc))
self._logger.debug("analyzer output: " + outstr)
except subprocess.CalledProcessError as err:
rc = err.returncode
outstr = err.output
outstr = outstr.decode('utf8')
if rc:
status = 'FAILED'
skip = True
success = False
self._logger.error("analyzer status: failed")
self._logger.error("analyzer exitcode: " + str(rc))
self._logger.error("analyzer output: " + outstr)
else:
status = 'SUCCESS'
else:
# this means that a prior analyzer failed, so we skip the rest
self._logger.debug(
"skipping analyzer (due to prior analyzer failure): "
+ script)
outstr = ""
rc = 1
status = 'SKIPPED'
mtype = "base"
if atype == 'user_scripts_dir':
mtype = 'user'
elif atype == 'extra_scripts_dir':
mtype = 'extra'
results[script] = {}
results[script]['command'] = cmdstr
results[script]['returncode'] = rc
results[script]['output'] = outstr
results[script]['outputdir'] = outputdir
results[script]['atype'] = atype
results[script]['csum'] = csum
results[script]['timestamp'] = time.time()
results[script]['status'] = status
if os.path.exists(
os.path.join(outputdir, 'analyzer_output')):
for d in os.listdir(
os.path.join(outputdir, 'analyzer_output')):
if os.path.exists(
os.path.join(outputdir, 'analyzer_output',
d)):
for dd in os.listdir(
os.path.join(outputdir,
'analyzer_output', d)):
module_name = d
module_value = dd
if 'analyzer_outputs' not in results[
script]:
#results[script]['analyzer_outputs'] = {}
results[script][
'analyzer_outputs'] = list()
aoutput = {
'module_name': module_name,
'module_value': module_value,
'module_type': mtype
}
if os.path.isdir(
os.path.join(
outputdir, 'analyzer_output',
d, dd)):
aoutput['data_type'] = 'dir'
else:
aoutput['data_type'] = 'file'
results[script]['analyzer_outputs'].append(
aoutput)
analyzer_status[script] = {}
analyzer_status[script].update(results[script])
else:
self._logger.debug(
"skipping analyzer (no change in analyzer/config and prior run succeeded): "
+ script)
# process and store analyzer outputs
didsave = False
for script in results.keys():
result = results[script]
if result['status'] == 'SUCCESS':
mtype = None
if result['atype'] == 'user_scripts_dir':
mtype = 'user'
elif result['atype'] == 'extra_scripts_dir':
mtype = 'extra'
if os.path.exists(
os.path.join(result['outputdir'], 'analyzer_output')):
for d in os.listdir(
os.path.join(result['outputdir'],
'analyzer_output')):
if os.path.exists(
os.path.join(result['outputdir'],
'analyzer_output', d)):
for dd in os.listdir(
os.path.join(result['outputdir'],
'analyzer_output', d)):
dfile = os.path.join(result['outputdir'],
'analyzer_output', d, dd)
module_name = d
module_value = dd
if os.path.isfile(dfile):
adata = anchore_utils.read_kvfile_todict(
dfile)
self.anchoreDB.save_analysis_output(
image.meta['imageId'],
module_name,
module_value,
adata,
module_type=mtype)
didsave = True
elif os.path.isdir(dfile):
self.anchoreDB.save_analysis_output(
image.meta['imageId'],
module_name,
module_value,
dfile,
module_type=mtype,
directory_data=True)
didsave = True
self.anchoreDB.save_analyzer_manifest(image.meta['imageId'],
analyzer_status)
if success:
self._logger.debug(
"analyzer commands all finished with successful exit codes")
if didsave:
self._logger.debug(
"generating analysis report from analyzer outputs and saving"
)
report = self.generate_analysis_report(image)
self.anchoreDB.save_analysis_report(image.meta['imageId'],
report)
self._logger.debug(
"saving image information with updated analysis data")
image.save_image()
self._logger.info(image.meta['shortId'] + ": analyzed.")
self._logger.debug("running analyzers on image: " +
str(image.meta['imagename']) + ": end")
return (success)
def run_analyzers(self, image):
success = True
analyzers = self.list_analyzers()
imagename = image.meta['imagename']
#outputdir = image.anchore_imagedir
shortid = image.meta['shortId']
imagedir = None
analyzer_status = self.anchoreDB.load_analyzer_manifest(image.meta['imageId'])
analyzer_config = {}
analyzer_config_csum = None
try:
analyzer_config, analyzer_config_csum = anchore_utils.load_analyzer_config(self.config.config_dir)
except:
pass
if 'analyzer_config_csum' in analyzer_status:
try:
if analyzer_status['analyzer_config_csum']['csum'] != analyzer_config_csum:
self._logger.debug("anchore analyzer config has been updating, forcing re-analysis")
self.force = True
analyzer_status['analyzer_config_csum']['csum'] = analyzer_config_csum
except:
pass
else:
script = 'analyzer_config_csum'
analyzer_status[script] = {}
analyzer_status[script]['command'] = "ANALYZER_CONFIG_META"
analyzer_status[script]['returncode'] = 0
analyzer_status[script]['output'] = ""
analyzer_status[script]['outputdir'] = ""
analyzer_status[script]['atype'] = 'base'
analyzer_status[script]['csum'] = analyzer_config_csum
analyzer_status[script]['timestamp'] = time.time()
analyzer_status[script]['status'] = 'SUCCESS'
results = {}
outputdirs = {}
torun = list()
skip = False
atypes = ['user_scripts_dir', 'extra_scripts_dir', 'base']
for atype in atypes:
for script in analyzers[atype]:
try:
with open(script, 'r') as FH:
csum = hashlib.md5(FH.read()).hexdigest()
except:
csum = "N/A"
# decide whether or not to run the analyzer
dorun = True
if self.force:
dorun = True
elif script in analyzer_status:
if csum == analyzer_status[script]['csum'] and analyzer_status[script]['returncode'] == 0:
dorun = False
outputdir = cmdstr = outstr = ""
if dorun:
if not skip:
if not imagedir:
self._logger.info(image.meta['shortId'] + ": analyzing ...")
imagedir = image.unpack()
if not imagedir:
self._logger.error("could not unpack image")
return(False)
outputdir = tempfile.mkdtemp(dir=imagedir)
cmdline = ' '.join([imagename, self.config['image_data_store'], outputdir, imagedir])
cmdstr = script + " " + cmdline
cmd = cmdstr.split()
try:
self._logger.debug("running analyzer: " + cmdstr)
timer = time.time()
outstr = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self._logger.debug("analyzer time (seconds): " + str(time.time() - timer))
rc = 0
self._logger.debug("analyzer status: success")
self._logger.debug("analyzer exitcode: " + str(rc))
self._logger.debug("analyzer output: " + outstr)
except subprocess.CalledProcessError as err:
rc = err.returncode
outstr = err.output
outstr = outstr.decode('utf8')
if rc:
status = 'FAILED'
skip = True
success = False
self._logger.error("analyzer status: failed")
self._logger.error("analyzer exitcode: " + str(rc))
self._logger.error("analyzer output: " + outstr)
else:
status = 'SUCCESS'
else:
# this means that a prior analyzer failed, so we skip the rest
self._logger.debug("skipping analyzer (due to prior analyzer failure): " + script)
outstr = ""
rc = 1
status = 'SKIPPED'
mtype = "base"
if atype == 'user_scripts_dir':
mtype = 'user'
elif atype == 'extra_scripts_dir':
mtype = 'extra'
results[script] = {}
results[script]['command'] = cmdstr
results[script]['returncode'] = rc
results[script]['output'] = outstr
results[script]['outputdir'] = outputdir
results[script]['atype'] = atype
results[script]['csum'] = csum
results[script]['timestamp'] = time.time()
results[script]['status'] = status
if os.path.exists(os.path.join(outputdir, 'analyzer_output')):
for d in os.listdir(os.path.join(outputdir, 'analyzer_output')):
if os.path.exists(os.path.join(outputdir, 'analyzer_output', d)):
for dd in os.listdir(os.path.join(outputdir, 'analyzer_output', d)):
module_name = d
module_value = dd
if 'analyzer_outputs' not in results[script]:
#results[script]['analyzer_outputs'] = {}
results[script]['analyzer_outputs'] = list()
aoutput = {'module_name':module_name, 'module_value':module_value, 'module_type':mtype}
if os.path.isdir(os.path.join(outputdir, 'analyzer_output', d, dd)):
aoutput['data_type'] = 'dir'
else:
aoutput['data_type'] = 'file'
results[script]['analyzer_outputs'].append(aoutput)
analyzer_status[script] = {}
analyzer_status[script].update(results[script])
else:
self._logger.debug("skipping analyzer (no change in analyzer/config and prior run succeeded): " + script)
# process and store analyzer outputs
didsave = False
for script in results.keys():
result = results[script]
if result['status'] == 'SUCCESS':
mtype = None
if result['atype'] == 'user_scripts_dir':
mtype = 'user'
elif result['atype'] == 'extra_scripts_dir':
mtype = 'extra'
if os.path.exists(os.path.join(result['outputdir'], 'analyzer_output')):
for d in os.listdir(os.path.join(result['outputdir'], 'analyzer_output')):
if os.path.exists(os.path.join(result['outputdir'], 'analyzer_output', d)):
for dd in os.listdir(os.path.join(result['outputdir'], 'analyzer_output', d)):
dfile = os.path.join(result['outputdir'], 'analyzer_output', d, dd)
module_name = d
module_value = dd
if os.path.isfile(dfile):
adata = anchore_utils.read_kvfile_todict(dfile)
self.anchoreDB.save_analysis_output(image.meta['imageId'], module_name, module_value, adata, module_type=mtype)
didsave = True
elif os.path.isdir(dfile):
self.anchoreDB.save_analysis_output(image.meta['imageId'], module_name, module_value, dfile, module_type=mtype, directory_data=True)
didsave = True
self.anchoreDB.save_analyzer_manifest(image.meta['imageId'], analyzer_status)
if success:
self._logger.debug("analyzer commands all finished with successful exit codes")
if didsave:
self._logger.debug("generating analysis report from analyzer outputs and saving")
report = self.generate_analysis_report(image)
self.anchoreDB.save_analysis_report(image.meta['imageId'], report)
self._logger.debug("saving image information with updated analysis data")
image.save_image()
self._logger.info(image.meta['shortId'] + ": analyzed.")
self._logger.debug("running analyzers on image: " + str(image.meta['imagename']) + ": end")
return(success)
