def build_image_list(config, image, imagefile, all_local, include_allanchore, dockerfile=None, exclude_file=None): """Given option inputs from the cli, construct a list of image ids. Includes all found with no exclusion logic""" if not image and not (imagefile or all_local): raise click.BadOptionUsage('No input found for image source. One of <image>, <imagefile>, or <all> must be specified') if image and imagefile: raise click.BadOptionUsage('Only one of <image> and <imagefile> can be specified') filter_images = [] if exclude_file: with open(exclude_file) as f: for line in f.readlines(): filter_images.append(line.strip()) imagelist = {} if image: imagelist[image] = {'dockerfile':dockerfile} if imagefile: filelist = anchore_utils.read_kvfile_tolist(imagefile) for i in range(len(filelist)): l = filelist[i] imageId = l[0] try: dfile = l[1] except: dfile = None imagelist[imageId] = {'dockerfile':dfile} if all_local: docker_cli = contexts['docker_cli'] if docker_cli: for f in docker_cli.images(all=True, quiet=True, filters={'dangling': False}): if f not in imagelist and f not in filter_images: imagelist[f] = {'dockerfile':None} else: raise Exception("Could not load any images from local docker host - is docker running?") if include_allanchore: ret = contexts['anchore_db'].load_all_images() if ret and len(ret) > 0: for l in list(set(imagelist.keys()) | set(ret)): imagelist[l] = {'dockerfile':None} # Remove excluded items for excluded in filter_images: docker_cli = contexts['docker_cli'] if not docker_cli: raise Exception("Could not query docker - is docker running?") for img in docker_cli.images(name=excluded, quiet=True): imagelist.pop(img, None) #imagelist.remove(img) return imagelist
def build_image_list(config, image, imagefile, all_local, include_allanchore, dockerfile=None, exclude_file=None): """Given option inputs from the cli, construct a list of image ids. Includes all found with no exclusion logic""" if not image and not (imagefile or all_local): raise click.BadOptionUsage('No input found for image source. One of <image>, <imagefile>, or <all> must be specified') if image and imagefile: raise click.BadOptionUsage('Only one of <image> and <imagefile> can be specified') filter_images = [] if exclude_file: with open(exclude_file) as f: for line in f.readlines(): filter_images.append(line.strip()) imagelist = {} if image: imagelist[image] = {'dockerfile':dockerfile} if imagefile: filelist = anchore_utils.read_kvfile_tolist(imagefile) for i in range(len(filelist)): l = filelist[i] imageId = l[0] try: dfile = l[1] except: dfile = None imagelist[imageId] = {'dockerfile':dfile} if all_local: docker_cli = contexts['docker_cli'] if docker_cli: for f in docker_cli.images(all=True, quiet=True, filters={'dangling': False}): if f not in imagelist and f not in filter_images: imagelist[f] = {'dockerfile':None} else: raise Exception("Could not load any images from local docker host - is docker running?") if include_allanchore: ret = list(contexts['anchore_db'].load_all_images().keys()) if ret and len(ret) > 0: for l in list(set(imagelist.keys()) | set(ret)): imagelist[l] = {'dockerfile':None} # Remove excluded items for excluded in filter_images: docker_cli = contexts['docker_cli'] if not docker_cli: raise Exception("Could not query docker - is docker running?") for img in docker_cli.images(name=excluded, quiet=True): imagelist.pop(img, None) #imagelist.remove(img) return imagelist
def load_global_whitelist(self): ret = [] whitelist_data = [] whitelist_file = None if self.global_whitelist_override and os.path.exists(self.global_whitelist_override): whitelist_file = self.global_whitelist_override elif self.default_global_whitelist and os.path.exists(self.default_global_whitelist): whitelist_file = self.default_global_whitelist else: self._logger.debug("no global whitelist can be found, skipping") if whitelist_file: whitelist_data = anchore_utils.read_kvfile_tolist(whitelist_file) for item in whitelist_data: if item[0] and not re.match("^#", item[0]) and len(item) > 1: store = item[0:2] ret.append(store) return(ret)
def execute_query(self, imglist, se, params): success = True datadir = self.config['image_data_store'] outputdir = '/'.join([ self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999)) ]) if not os.path.exists(outputdir): os.makedirs(outputdir) imgfile = '/'.join([ self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999)) ]) anchore_utils.write_plainfile_fromlist(imgfile, imglist) cmdline = ' '.join([imgfile, datadir, outputdir]) if params: cmdline = cmdline + ' ' + ' '.join(params) meta = {} try: (cmd, rc, sout) = se.execute(capture_output=True, cmdline=cmdline) if rc: self._logger.error("Query command ran but execution failed") self._logger.error("Query command: (" + ' '.join([se.thecmd, cmdline]) + ")") self._logger.error("Query output: (" + str(sout) + ")") self._logger.error("Exit code: (" + str(rc) + ")") raise Exception("Query ran but exited non-zero.") except Exception as err: raise Exception("Query execution failed: " + str(err)) else: try: #outputs = os.listdir(outputdir) warnfile = False found = False for f in os.listdir(outputdir): if re.match(".*\.WARNS", f): warnfile = '/'.join([outputdir, f]) else: ofile = '/'.join([outputdir, f]) found = True if not found: raise Exception( "No output files found after executing query command\n\tCommand Output:\n" + sout + "\n\tInfo: Query command should have produced an output file in: " + outputdir) orows = list() try: frows = anchore_utils.read_kvfile_tolist(ofile) header = frows[0] rowlen = len(header) for row in frows[1:]: if len(row) != rowlen: raise Exception( "Number of columns in data row (" + str(len(row)) + ") is not equal to number of columns in header (" + str(rowlen) + ")\n\tHeader: " + str(header) + "\n\tOffending Row: " + str(row)) orows.append(row) except Exception as err: raise err if warnfile: try: meta['warns'] = anchore_utils.read_plainfile_tolist( warnfile) except: pass meta['queryparams'] = ','.join(params) meta['querycommand'] = cmd try: i = header.index('URL') meta['url_column_index'] = i except: pass meta['result'] = {} meta['result']['header'] = header meta['result']['rowcount'] = len(orows) try: #meta['result']['colcount'] = len(orows[0]) meta['result']['colcount'] = len(header) except: meta['result']['colcount'] = 0 meta['result']['rows'] = orows except Exception as err: self._logger.error("Query output handling failed: ") self._logger.error("\tCommand: " + str(cmd)) self._logger.error("\tException: " + str(err)) success = False finally: if imgfile and os.path.exists(imgfile): os.remove(imgfile) if outputdir and os.path.exists(outputdir): shutil.rmtree(outputdir) ret = [success, cmd, meta] return (ret)
def execute_query(self, imglist, qcommand, params): success = True datadir = self.config['image_data_store'] outputdir = '/'.join([ self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999)) ]) os.makedirs(outputdir) imgfile = '/'.join([ self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999)) ]) anchore_utils.write_plainfile_fromlist(imgfile, imglist) cmd = [qcommand, imgfile, datadir, outputdir] if params: cmd = cmd + params meta = {} try: self._logger.debug("Running query command: " + str(' '.join(cmd))) sout = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self._logger.debug("Query command execution success") if sout: self._logger.debug("\tCommand output:\n" + str(sout)) except subprocess.CalledProcessError as err: self._logger.error("Query command ran but execution failed: ") self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) self._logger.error("\tCommand output:\n" + str(err.output)) self._logger.error("\tExit code: " + str(err.returncode)) success = False except Exception as err: self._logger.error("Query command execution call failed: ") self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) success = False else: try: outputs = os.listdir(outputdir) if len(outputs) <= 0: raise Exception( "No output files found after executing query command\n\tCommand Output:\n" + sout + "\n\tInfo: Query command should have produced an output file in: " + outputdir) orows = list() ofile = outputs[0] try: frows = anchore_utils.read_kvfile_tolist('/'.join( [outputdir, ofile])) header = frows[0] rowlen = len(header) for row in frows[1:]: if len(row) != rowlen: raise Exception( "Number of columns in data row (" + str(len(row)) + ") is not equal to number of columns in header (" + str(rowlen) + ")\n\tHeader: " + str(header) + "\n\tOffending Row: " + str(row)) orows.append(row) except Exception as err: raise err meta['queryparams'] = ','.join(params) meta['querycommand'] = cmd meta['result'] = {} meta['result']['header'] = header meta['result']['rowcount'] = len(orows) try: meta['result']['colcount'] = len(orows[0]) except: meta['result']['colcount'] = 0 meta['result']['rows'] = orows except Exception as err: self._logger.error("Query output handling failed: ") self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) success = False finally: os.remove(imgfile) ret = [success, cmd, outputdir, meta] return (ret)
def execute_query(self, imglist, qcommand, params): success = True datadir = self.config['image_data_store'] outputdir = '/'.join([self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999))]) os.makedirs(outputdir) imgfile = '/'.join([self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999))]) anchore_utils.write_plainfile_fromlist(imgfile, imglist) cmd = [qcommand, imgfile, datadir, outputdir] if params: cmd = cmd + params meta = {} try: self._logger.debug("Running query command: " + str(' '.join(cmd))) sout = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self._logger.debug("Query command execution success") if sout: self._logger.debug("\tCommand output:\n" + str(sout)) except subprocess.CalledProcessError as err: self._logger.error("Query command ran but execution failed: " ) self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) self._logger.error("\tCommand output:\n" + str(err.output)) self._logger.error("\tExit code: " + str(err.returncode)) success = False except Exception as err: self._logger.error("Query command execution call failed: " ) self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) success = False else: try: outputs = os.listdir(outputdir) if len(outputs) <= 0: raise Exception("No output files found after executing query command\n\tCommand Output:\n"+sout+"\n\tInfo: Query command should have produced an output file in: " + outputdir) orows = list() ofile = outputs[0] try: frows = anchore_utils.read_kvfile_tolist('/'.join([outputdir, ofile])) header = frows[0] rowlen = len(header) for row in frows[1:]: if len(row) != rowlen: raise Exception("Number of columns in data row ("+str(len(row))+") is not equal to number of columns in header ("+str(rowlen)+")\n\tHeader: "+str(header)+"\n\tOffending Row: "+str(row)) orows.append(row) except Exception as err: raise err meta['queryparams'] = ','.join(params) meta['querycommand'] = cmd meta['result'] = {} meta['result']['header'] = header meta['result']['rowcount'] = len(orows) try: meta['result']['colcount'] = len(orows[0]) except: meta['result']['colcount'] = 0 meta['result']['rows'] = orows except Exception as err: self._logger.error("Query output handling failed: ") self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) success = False finally: os.remove(imgfile) ret = [success, cmd, outputdir, meta] return(ret)
def execute_query(self, imglist, se, params): success = True datadir = self.config['image_data_store'] outputdir = '/'.join([self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999))]) if not os.path.exists(outputdir): os.makedirs(outputdir) imgfile = '/'.join([self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999))]) anchore_utils.write_plainfile_fromlist(imgfile, imglist) cmdline = ' '.join([imgfile, datadir, outputdir]) if params: cmdline = cmdline + ' ' + ' '.join(params) meta = {} try: (cmd, rc, sout) = se.execute(capture_output=True, cmdline=cmdline) if rc: self._logger.error("Query command ran but execution failed" ) self._logger.error("Query command: (" + ' '.join([se.thecmd, cmdline])+")") self._logger.error("Query output: (" + str(sout) + ")") self._logger.error("Exit code: (" + str(rc)+")") raise Exception("Query ran but exited non-zero.") except Exception as err: raise Exception("Query execution failed: " + str(err)) else: try: #outputs = os.listdir(outputdir) warnfile = False found = False for f in os.listdir(outputdir): if re.match(".*\.WARNS", f): warnfile = '/'.join([outputdir, f]) else: ofile = '/'.join([outputdir, f]) found=True if not found: raise Exception("No output files found after executing query command\n\tCommand Output:\n"+sout+"\n\tInfo: Query command should have produced an output file in: " + outputdir) orows = list() try: frows = anchore_utils.read_kvfile_tolist(ofile) header = frows[0] rowlen = len(header) for row in frows[1:]: if len(row) != rowlen: raise Exception("Number of columns in data row ("+str(len(row))+") is not equal to number of columns in header ("+str(rowlen)+")\n\tHeader: "+str(header)+"\n\tOffending Row: "+str(row)) orows.append(row) except Exception as err: raise err if warnfile: try: meta['warns'] = anchore_utils.read_plainfile_tolist(warnfile) except: pass meta['queryparams'] = ','.join(params) meta['querycommand'] = cmd try: i = header.index('URL') meta['url_column_index'] = i except: pass meta['result'] = {} meta['result']['header'] = header meta['result']['rowcount'] = len(orows) try: #meta['result']['colcount'] = len(orows[0]) meta['result']['colcount'] = len(header) except: meta['result']['colcount'] = 0 meta['result']['rows'] = orows except Exception as err: self._logger.error("Query output handling failed: ") self._logger.error("\tCommand: " + str(cmd)) self._logger.error("\tException: " + str(err)) success = False finally: if imgfile and os.path.exists(imgfile): os.remove(imgfile) if outputdir and os.path.exists(outputdir): shutil.rmtree(outputdir) ret = [success, cmd, meta] return(ret)