def run_anchore_analyzers(staging_dirs, imageDigest, imageId): outputdir = staging_dirs['outputdir'] unpackdir = staging_dirs['unpackdir'] copydir = staging_dirs['copydir'] # run analyzers anchore_module_root = resource_filename("anchore", "anchore-modules") analyzer_root = os.path.join(anchore_module_root, "analyzers") for f in os.listdir(analyzer_root): thecmd = os.path.join(analyzer_root, f) if re.match(".*\.py$", thecmd): cmdstr = " ".join( [thecmd, imageId, unpackdir, outputdir, unpackdir]) if True: try: rc, sout, serr = anchore_engine.services.common.run_command( cmdstr) if rc != 0: raise Exception("command failed: cmd=" + str(cmdstr) + " exitcode=" + str(rc) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) else: logger.debug("command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(serr).strip()) except Exception as err: logger.error("command failed with exception - " + str(err)) #raise err analyzer_manifest = {} #TODO populate analyzer_manifest? analyzer_report = {} for analyzer_output in os.listdir( os.path.join(outputdir, "analyzer_output")): if analyzer_output not in analyzer_report: analyzer_report[analyzer_output] = {} for analyzer_output_el in os.listdir( os.path.join(outputdir, "analyzer_output", analyzer_output)): if analyzer_output_el not in analyzer_report[analyzer_output]: analyzer_report[analyzer_output][analyzer_output_el] = { 'base': {} } data = read_kvfile_todict( os.path.join(outputdir, "analyzer_output", analyzer_output, analyzer_output_el)) if data: analyzer_report[analyzer_output][analyzer_output_el][ 'base'] = read_kvfile_todict( os.path.join(outputdir, "analyzer_output", analyzer_output, analyzer_output_el)) else: analyzer_report[analyzer_output].pop(analyzer_output_el, None) if not analyzer_report[analyzer_output]: analyzer_report.pop(analyzer_output, None) return (analyzer_report)
def load_analysis_output(self, imageId, module_name, module_value, module_type=None): ret = {} if not module_type or module_type == 'base': thefile = '/'.join([self.imagerootdir, imageId, "analyzer_output", module_name, module_value]) else: thefile = '/'.join([self.imagerootdir, imageId, "analyzer_output_"+module_type, module_name, module_value]) if os.path.exists(thefile): if os.path.isfile(thefile): ret = anchore_utils.read_kvfile_todict(thefile) elif os.path.isdir(thefile): ret = thefile import tarfile, io try: TFH = io.BytesIO() tar = tarfile.open(fileobj=TFH, mode='w:gz', format=tarfile.PAX_FORMAT) for l in os.listdir(thefile): tarfile = os.path.join(thefile, l) tar.add(tarfile) tar.close() TFH.seek(0) ret = TFH except Exception as err: import traceback traceback.print_exc() print str(err) return(ret)
def run_analyzers(self, image): success = True analyzers = self.list_analyzers() imagename = image.meta['imagename'] outputdir = image.anchore_imagedir shortid = image.meta['shortId'] imagedir = None analyzer_status = self.anchoreDB.load_analyzer_manifest( image.meta['imageId']) results = {} outputdirs = {} torun = list() skip = False for atype in ['user_scripts_dir', 'extra_scripts_dir', 'base']: for script in analyzers[atype]: try: with open(script, 'r') as FH: csum = hashlib.md5(FH.read()).hexdigest() except: csum = "N/A" # decide whether or not to run the analyzer dorun = True if self.force: dorun = True elif script in analyzer_status: if csum == analyzer_status[script][ 'csum'] and analyzer_status[script][ 'returncode'] == 0: dorun = False outputdir = cmdstr = outstr = "" if dorun: if not skip: if not imagedir: self._logger.info(image.meta['shortId'] + ": analyzing ...") imagedir = image.unpack() outputdir = tempfile.mkdtemp(dir=imagedir) cmdline = ' '.join([ imagename, self.config['image_data_store'], outputdir, imagedir ]) cmdstr = script + " " + cmdline cmd = cmdstr.split() try: self._logger.debug("running analyzer: " + cmdstr) timer = time.time() outstr = subprocess.check_output( cmd, stderr=subprocess.STDOUT) self._logger.debug("analyzer time (seconds): " + str(time.time() - timer)) rc = 0 self._logger.debug("analyzer status: success") self._logger.debug("analyzer exitcode: " + str(rc)) self._logger.debug("analyzer output: " + outstr) except subprocess.CalledProcessError as err: rc = err.returncode outstr = err.output outstr = outstr.decode('utf8') if rc: status = 'FAILED' skip = True success = False self._logger.error("analyzer status: failed") self._logger.error("analyzer exitcode: " + str(rc)) self._logger.error("analyzer output: " + outstr) else: status = 'SUCCESS' else: # this means that a prior analyzer failed, so we skip the rest self._logger.debug( "skipping analyzer (due to prior analyzer failure): " + script) outstr = "" rc = 1 status = 'SKIPPED' mtype = "base" if atype == 'user_scripts_dir': mtype = 'user' elif atype == 'extra_scripts_dir': mtype = 'extra' results[script] = {} results[script]['command'] = cmdstr results[script]['returncode'] = rc results[script]['output'] = outstr results[script]['outputdir'] = outputdir results[script]['atype'] = atype results[script]['csum'] = csum results[script]['timestamp'] = time.time() results[script]['status'] = status if os.path.exists( os.path.join(outputdir, 'analyzer_output')): for d in os.listdir( os.path.join(outputdir, 'analyzer_output')): if os.path.exists( os.path.join(outputdir, 'analyzer_output', d)): for dd in os.listdir( os.path.join(outputdir, 'analyzer_output', d)): module_name = d module_value = dd if 'analyzer_outputs' not in results[ script]: #results[script]['analyzer_outputs'] = {} results[script][ 'analyzer_outputs'] = list() aoutput = { 'module_name': module_name, 'module_value': module_value, 'module_type': mtype } if os.path.isdir( os.path.join( outputdir, 'analyzer_output', d, dd)): aoutput['data_type'] = 'dir' else: aoutput['data_type'] = 'file' results[script]['analyzer_outputs'].append( aoutput) analyzer_status[script] = {} analyzer_status[script].update(results[script]) else: self._logger.debug( "skipping analyzer (no change in analyzer/config and prior run succeeded): " + script) # process and store analyzer outputs didsave = False for script in results.keys(): result = results[script] if result['status'] == 'SUCCESS': mtype = None if result['atype'] == 'user_scripts_dir': mtype = 'user' elif result['atype'] == 'extra_scripts_dir': mtype = 'extra' if os.path.exists( os.path.join(result['outputdir'], 'analyzer_output')): for d in os.listdir( os.path.join(result['outputdir'], 'analyzer_output')): if os.path.exists( os.path.join(result['outputdir'], 'analyzer_output', d)): for dd in os.listdir( os.path.join(result['outputdir'], 'analyzer_output', d)): dfile = os.path.join(result['outputdir'], 'analyzer_output', d, dd) module_name = d module_value = dd if os.path.isfile(dfile): adata = anchore_utils.read_kvfile_todict( dfile) self.anchoreDB.save_analysis_output( image.meta['imageId'], module_name, module_value, adata, module_type=mtype) didsave = True elif os.path.isdir(dfile): self.anchoreDB.save_analysis_output( image.meta['imageId'], module_name, module_value, dfile, module_type=mtype, directory_data=True) didsave = True self.anchoreDB.save_analyzer_manifest(image.meta['imageId'], analyzer_status) if success: self._logger.debug( "analyzer commands all finished with successful exit codes") if didsave: self._logger.debug( "generating analysis report from analyzer outputs and saving" ) report = self.generate_analysis_report(image) self.anchoreDB.save_analysis_report(image.meta['imageId'], report) self._logger.debug( "saving image information with updated analysis data") image.save_image() self._logger.info(image.meta['shortId'] + ": analyzed.") self._logger.debug("running analyzers on image: " + str(image.meta['imagename']) + ": end") return (success)
def run_analyzers(self, image): success = True analyzers = self.list_analyzers() imagename = image.meta['imagename'] #outputdir = image.anchore_imagedir shortid = image.meta['shortId'] imagedir = None analyzer_status = self.anchoreDB.load_analyzer_manifest(image.meta['imageId']) analyzer_config = {} analyzer_config_csum = None try: analyzer_config, analyzer_config_csum = anchore_utils.load_analyzer_config(self.config.config_dir) except: pass if 'analyzer_config_csum' in analyzer_status: try: if analyzer_status['analyzer_config_csum']['csum'] != analyzer_config_csum: self._logger.debug("anchore analyzer config has been updating, forcing re-analysis") self.force = True analyzer_status['analyzer_config_csum']['csum'] = analyzer_config_csum except: pass else: script = 'analyzer_config_csum' analyzer_status[script] = {} analyzer_status[script]['command'] = "ANALYZER_CONFIG_META" analyzer_status[script]['returncode'] = 0 analyzer_status[script]['output'] = "" analyzer_status[script]['outputdir'] = "" analyzer_status[script]['atype'] = 'base' analyzer_status[script]['csum'] = analyzer_config_csum analyzer_status[script]['timestamp'] = time.time() analyzer_status[script]['status'] = 'SUCCESS' results = {} outputdirs = {} torun = list() skip = False atypes = ['user_scripts_dir', 'extra_scripts_dir', 'base'] for atype in atypes: for script in analyzers[atype]: try: with open(script, 'r') as FH: csum = hashlib.md5(FH.read()).hexdigest() except: csum = "N/A" # decide whether or not to run the analyzer dorun = True if self.force: dorun = True elif script in analyzer_status: if csum == analyzer_status[script]['csum'] and analyzer_status[script]['returncode'] == 0: dorun = False outputdir = cmdstr = outstr = "" if dorun: if not skip: if not imagedir: self._logger.info(image.meta['shortId'] + ": analyzing ...") imagedir = image.unpack() if not imagedir: self._logger.error("could not unpack image") return(False) outputdir = tempfile.mkdtemp(dir=imagedir) cmdline = ' '.join([imagename, self.config['image_data_store'], outputdir, imagedir]) cmdstr = script + " " + cmdline cmd = cmdstr.split() try: self._logger.debug("running analyzer: " + cmdstr) timer = time.time() outstr = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self._logger.debug("analyzer time (seconds): " + str(time.time() - timer)) rc = 0 self._logger.debug("analyzer status: success") self._logger.debug("analyzer exitcode: " + str(rc)) self._logger.debug("analyzer output: " + outstr) except subprocess.CalledProcessError as err: rc = err.returncode outstr = err.output outstr = outstr.decode('utf8') if rc: status = 'FAILED' skip = True success = False self._logger.error("analyzer status: failed") self._logger.error("analyzer exitcode: " + str(rc)) self._logger.error("analyzer output: " + outstr) else: status = 'SUCCESS' else: # this means that a prior analyzer failed, so we skip the rest self._logger.debug("skipping analyzer (due to prior analyzer failure): " + script) outstr = "" rc = 1 status = 'SKIPPED' mtype = "base" if atype == 'user_scripts_dir': mtype = 'user' elif atype == 'extra_scripts_dir': mtype = 'extra' results[script] = {} results[script]['command'] = cmdstr results[script]['returncode'] = rc results[script]['output'] = outstr results[script]['outputdir'] = outputdir results[script]['atype'] = atype results[script]['csum'] = csum results[script]['timestamp'] = time.time() results[script]['status'] = status if os.path.exists(os.path.join(outputdir, 'analyzer_output')): for d in os.listdir(os.path.join(outputdir, 'analyzer_output')): if os.path.exists(os.path.join(outputdir, 'analyzer_output', d)): for dd in os.listdir(os.path.join(outputdir, 'analyzer_output', d)): module_name = d module_value = dd if 'analyzer_outputs' not in results[script]: #results[script]['analyzer_outputs'] = {} results[script]['analyzer_outputs'] = list() aoutput = {'module_name':module_name, 'module_value':module_value, 'module_type':mtype} if os.path.isdir(os.path.join(outputdir, 'analyzer_output', d, dd)): aoutput['data_type'] = 'dir' else: aoutput['data_type'] = 'file' results[script]['analyzer_outputs'].append(aoutput) analyzer_status[script] = {} analyzer_status[script].update(results[script]) else: self._logger.debug("skipping analyzer (no change in analyzer/config and prior run succeeded): " + script) # process and store analyzer outputs didsave = False for script in results.keys(): result = results[script] if result['status'] == 'SUCCESS': mtype = None if result['atype'] == 'user_scripts_dir': mtype = 'user' elif result['atype'] == 'extra_scripts_dir': mtype = 'extra' if os.path.exists(os.path.join(result['outputdir'], 'analyzer_output')): for d in os.listdir(os.path.join(result['outputdir'], 'analyzer_output')): if os.path.exists(os.path.join(result['outputdir'], 'analyzer_output', d)): for dd in os.listdir(os.path.join(result['outputdir'], 'analyzer_output', d)): dfile = os.path.join(result['outputdir'], 'analyzer_output', d, dd) module_name = d module_value = dd if os.path.isfile(dfile): adata = anchore_utils.read_kvfile_todict(dfile) self.anchoreDB.save_analysis_output(image.meta['imageId'], module_name, module_value, adata, module_type=mtype) didsave = True elif os.path.isdir(dfile): self.anchoreDB.save_analysis_output(image.meta['imageId'], module_name, module_value, dfile, module_type=mtype, directory_data=True) didsave = True self.anchoreDB.save_analyzer_manifest(image.meta['imageId'], analyzer_status) if success: self._logger.debug("analyzer commands all finished with successful exit codes") if didsave: self._logger.debug("generating analysis report from analyzer outputs and saving") report = self.generate_analysis_report(image) self.anchoreDB.save_analysis_report(image.meta['imageId'], report) self._logger.debug("saving image information with updated analysis data") image.save_image() self._logger.info(image.meta['shortId'] + ": analyzed.") self._logger.debug("running analyzers on image: " + str(image.meta['imagename']) + ": end") return(success)