def discover_gates(self): ret = {} dirmap = { '/'.join([self.config["scripts_dir"], "gates"]):'base', '/'.join([self.config['user_scripts_dir'], 'gates']):'user' } if self.config['extra_scripts_dir']: dirmap['/'.join([self.config['extra_scripts_dir'], 'gates'])] = 'xtra' for gdir in dirmap.keys(): gtype = dirmap[gdir] gatesdir = gdir for f in os.listdir(gatesdir): thegate = '/'.join([gatesdir, f]) if os.access(thegate, os.R_OK ^ os.X_OK): tmpdir = anchore_utils.make_anchoretmpdir(self.config['tmpdir']) namedir = '/'.join([tmpdir, 'namedir']) os.makedirs(namedir) imgfile = '/'.join([tmpdir, 'images']) anchore_utils.write_plainfile_fromlist(imgfile, self.images) cmdobj = scripting.ScriptExecutor(path=gatesdir, script_name=f) out = cmdobj.execute(capture_output=True, cmdline=' '.join([imgfile,self.config['image_data_store'], namedir, 'anchore_getname'])) for ff in os.listdir(namedir): if gtype not in ret: ret[gtype] = {} ret[gtype][f] = ff shutil.rmtree(tmpdir) return(ret)
def edit_policy_file(self, editpolicy=False, whitelist=False): ret = True if not editpolicy and not whitelist: # nothing to do return (ret) for imageId in self.images: if editpolicy: data = self.anchoreDB.load_gate_policy(imageId) else: data = self.anchoreDB.load_gate_whitelist(imageId) if not data: self._logger.info( "Cannot find existing data to edit, skipping: " + str(imageId)) else: tmpdir = anchore_utils.make_anchoretmpdir("/tmp") try: thefile = os.path.join(tmpdir, "anchorepol." + imageId) anchore_utils.write_plainfile_fromlist(thefile, data) if "EDITOR" in os.environ: cmd = os.environ["EDITOR"].split() cmd.append(thefile) try: subprocess.check_output(cmd, shell=False) except: ret = False elif os.path.exists("/bin/vi"): try: rc = os.system("/bin/vi " + thefile) if rc: ret = False except: ret = False else: self._logger.info( "Cannot find editor to use: please set the EDITOR environment variable and try again" ) break ret = False newdata = anchore_utils.read_plainfile_tolist(thefile) if editpolicy: self.anchoreDB.save_gate_policy(imageId, newdata) else: self.anchoreDB.save_gate_whitelist(imageId, newdata) except Exception as err: pass finally: if tmpdir: shutil.rmtree(tmpdir) return (ret)
def save_gate_whitelist(self, imageId, data): thedir = os.path.join(self.imagerootdir, imageId) if not os.path.exists(thedir): os.makedirs(thedir) thefile = os.path.join(thedir, 'anchore_gate.whitelist') return (anchore_utils.write_plainfile_fromlist(thefile, data))
def save_gate_eval_output(self, imageId, gate_name, data): thedir = os.path.join(self.imagerootdir, imageId, "gates_output") if not os.path.exists(thedir): os.makedirs(thedir) thefile = os.path.join(thedir, gate_name + ".eval") return (anchore_utils.write_plainfile_fromlist(thefile, data))
def save_gate_whitelist(self, imageId, data): thedir = os.path.join(self.imagerootdir, imageId) if not os.path.exists(thedir): os.makedirs(thedir) thefile = os.path.join(thedir, 'anchore_gate.whitelist') return(anchore_utils.write_plainfile_fromlist(thefile, data))
def save_gate_eval_output(self, imageId, gate_name, data): thedir = os.path.join(self.imagerootdir, imageId, "gates_output") if not os.path.exists(thedir): os.makedirs(thedir) thefile = os.path.join(thedir, gate_name+".eval") return(anchore_utils.write_plainfile_fromlist(thefile, data))
def execute_query(self, imglist, se, params): success = True datadir = self.config['image_data_store'] outputdir = '/'.join([ self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999)) ]) if not os.path.exists(outputdir): os.makedirs(outputdir) imgfile = '/'.join([ self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999)) ]) anchore_utils.write_plainfile_fromlist(imgfile, imglist) cmdline = ' '.join([imgfile, datadir, outputdir]) if params: cmdline = cmdline + ' ' + ' '.join(params) meta = {} try: (cmd, rc, sout) = se.execute(capture_output=True, cmdline=cmdline) if rc: self._logger.error("Query command ran but execution failed") self._logger.error("Query command: (" + ' '.join([se.thecmd, cmdline]) + ")") self._logger.error("Query output: (" + str(sout) + ")") self._logger.error("Exit code: (" + str(rc) + ")") raise Exception("Query ran but exited non-zero.") except Exception as err: raise Exception("Query execution failed: " + str(err)) else: try: #outputs = os.listdir(outputdir) warnfile = False found = False for f in os.listdir(outputdir): if re.match(".*\.WARNS", f): warnfile = '/'.join([outputdir, f]) else: ofile = '/'.join([outputdir, f]) found = True if not found: raise Exception( "No output files found after executing query command\n\tCommand Output:\n" + sout + "\n\tInfo: Query command should have produced an output file in: " + outputdir) orows = list() try: frows = anchore_utils.read_kvfile_tolist(ofile) header = frows[0] rowlen = len(header) for row in frows[1:]: if len(row) != rowlen: raise Exception( "Number of columns in data row (" + str(len(row)) + ") is not equal to number of columns in header (" + str(rowlen) + ")\n\tHeader: " + str(header) + "\n\tOffending Row: " + str(row)) orows.append(row) except Exception as err: raise err if warnfile: try: meta['warns'] = anchore_utils.read_plainfile_tolist( warnfile) except: pass meta['queryparams'] = ','.join(params) meta['querycommand'] = cmd try: i = header.index('URL') meta['url_column_index'] = i except: pass meta['result'] = {} meta['result']['header'] = header meta['result']['rowcount'] = len(orows) try: #meta['result']['colcount'] = len(orows[0]) meta['result']['colcount'] = len(header) except: meta['result']['colcount'] = 0 meta['result']['rows'] = orows except Exception as err: self._logger.error("Query output handling failed: ") self._logger.error("\tCommand: " + str(cmd)) self._logger.error("\tException: " + str(err)) success = False finally: if imgfile and os.path.exists(imgfile): os.remove(imgfile) if outputdir and os.path.exists(outputdir): shutil.rmtree(outputdir) ret = [success, cmd, meta] return (ret)
def execute_query(self, imglist, qcommand, params): success = True datadir = self.config['image_data_store'] outputdir = '/'.join([ self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999)) ]) os.makedirs(outputdir) imgfile = '/'.join([ self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999)) ]) anchore_utils.write_plainfile_fromlist(imgfile, imglist) cmd = [qcommand, imgfile, datadir, outputdir] if params: cmd = cmd + params meta = {} try: self._logger.debug("Running query command: " + str(' '.join(cmd))) sout = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self._logger.debug("Query command execution success") if sout: self._logger.debug("\tCommand output:\n" + str(sout)) except subprocess.CalledProcessError as err: self._logger.error("Query command ran but execution failed: ") self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) self._logger.error("\tCommand output:\n" + str(err.output)) self._logger.error("\tExit code: " + str(err.returncode)) success = False except Exception as err: self._logger.error("Query command execution call failed: ") self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) success = False else: try: outputs = os.listdir(outputdir) if len(outputs) <= 0: raise Exception( "No output files found after executing query command\n\tCommand Output:\n" + sout + "\n\tInfo: Query command should have produced an output file in: " + outputdir) orows = list() ofile = outputs[0] try: frows = anchore_utils.read_kvfile_tolist('/'.join( [outputdir, ofile])) header = frows[0] rowlen = len(header) for row in frows[1:]: if len(row) != rowlen: raise Exception( "Number of columns in data row (" + str(len(row)) + ") is not equal to number of columns in header (" + str(rowlen) + ")\n\tHeader: " + str(header) + "\n\tOffending Row: " + str(row)) orows.append(row) except Exception as err: raise err meta['queryparams'] = ','.join(params) meta['querycommand'] = cmd meta['result'] = {} meta['result']['header'] = header meta['result']['rowcount'] = len(orows) try: meta['result']['colcount'] = len(orows[0]) except: meta['result']['colcount'] = 0 meta['result']['rows'] = orows except Exception as err: self._logger.error("Query output handling failed: ") self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) success = False finally: os.remove(imgfile) ret = [success, cmd, outputdir, meta] return (ret)
def execute_query(self, imglist, qcommand, params): success = True datadir = self.config['image_data_store'] outputdir = '/'.join([self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999))]) os.makedirs(outputdir) imgfile = '/'.join([self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999))]) anchore_utils.write_plainfile_fromlist(imgfile, imglist) cmd = [qcommand, imgfile, datadir, outputdir] if params: cmd = cmd + params meta = {} try: self._logger.debug("Running query command: " + str(' '.join(cmd))) sout = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self._logger.debug("Query command execution success") if sout: self._logger.debug("\tCommand output:\n" + str(sout)) except subprocess.CalledProcessError as err: self._logger.error("Query command ran but execution failed: " ) self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) self._logger.error("\tCommand output:\n" + str(err.output)) self._logger.error("\tExit code: " + str(err.returncode)) success = False except Exception as err: self._logger.error("Query command execution call failed: " ) self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) success = False else: try: outputs = os.listdir(outputdir) if len(outputs) <= 0: raise Exception("No output files found after executing query command\n\tCommand Output:\n"+sout+"\n\tInfo: Query command should have produced an output file in: " + outputdir) orows = list() ofile = outputs[0] try: frows = anchore_utils.read_kvfile_tolist('/'.join([outputdir, ofile])) header = frows[0] rowlen = len(header) for row in frows[1:]: if len(row) != rowlen: raise Exception("Number of columns in data row ("+str(len(row))+") is not equal to number of columns in header ("+str(rowlen)+")\n\tHeader: "+str(header)+"\n\tOffending Row: "+str(row)) orows.append(row) except Exception as err: raise err meta['queryparams'] = ','.join(params) meta['querycommand'] = cmd meta['result'] = {} meta['result']['header'] = header meta['result']['rowcount'] = len(orows) try: meta['result']['colcount'] = len(orows[0]) except: meta['result']['colcount'] = 0 meta['result']['rows'] = orows except Exception as err: self._logger.error("Query output handling failed: ") self._logger.error("\tCommand: " + ' '.join(cmd)) self._logger.error("\tException: " + str(err)) success = False finally: os.remove(imgfile) ret = [success, cmd, outputdir, meta] return(ret)
def execute_query(self, imglist, se, params): success = True datadir = self.config['image_data_store'] outputdir = '/'.join([self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999))]) if not os.path.exists(outputdir): os.makedirs(outputdir) imgfile = '/'.join([self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999))]) anchore_utils.write_plainfile_fromlist(imgfile, imglist) cmdline = ' '.join([imgfile, datadir, outputdir]) if params: cmdline = cmdline + ' ' + ' '.join(params) meta = {} try: (cmd, rc, sout) = se.execute(capture_output=True, cmdline=cmdline) if rc: self._logger.error("Query command ran but execution failed" ) self._logger.error("Query command: (" + ' '.join([se.thecmd, cmdline])+")") self._logger.error("Query output: (" + str(sout) + ")") self._logger.error("Exit code: (" + str(rc)+")") raise Exception("Query ran but exited non-zero.") except Exception as err: raise Exception("Query execution failed: " + str(err)) else: try: #outputs = os.listdir(outputdir) warnfile = False found = False for f in os.listdir(outputdir): if re.match(".*\.WARNS", f): warnfile = '/'.join([outputdir, f]) else: ofile = '/'.join([outputdir, f]) found=True if not found: raise Exception("No output files found after executing query command\n\tCommand Output:\n"+sout+"\n\tInfo: Query command should have produced an output file in: " + outputdir) orows = list() try: frows = anchore_utils.read_kvfile_tolist(ofile) header = frows[0] rowlen = len(header) for row in frows[1:]: if len(row) != rowlen: raise Exception("Number of columns in data row ("+str(len(row))+") is not equal to number of columns in header ("+str(rowlen)+")\n\tHeader: "+str(header)+"\n\tOffending Row: "+str(row)) orows.append(row) except Exception as err: raise err if warnfile: try: meta['warns'] = anchore_utils.read_plainfile_tolist(warnfile) except: pass meta['queryparams'] = ','.join(params) meta['querycommand'] = cmd try: i = header.index('URL') meta['url_column_index'] = i except: pass meta['result'] = {} meta['result']['header'] = header meta['result']['rowcount'] = len(orows) try: #meta['result']['colcount'] = len(orows[0]) meta['result']['colcount'] = len(header) except: meta['result']['colcount'] = 0 meta['result']['rows'] = orows except Exception as err: self._logger.error("Query output handling failed: ") self._logger.error("\tCommand: " + str(cmd)) self._logger.error("\tException: " + str(err)) success = False finally: if imgfile and os.path.exists(imgfile): os.remove(imgfile) if outputdir and os.path.exists(outputdir): shutil.rmtree(outputdir) ret = [success, cmd, meta] return(ret)