def discover_gates(self):
ret = {}
dirmap = {
'/'.join([self.config["scripts_dir"], "gates"]):'base',
'/'.join([self.config['user_scripts_dir'], 'gates']):'user'
}
if self.config['extra_scripts_dir']:
dirmap['/'.join([self.config['extra_scripts_dir'], 'gates'])] = 'xtra'
for gdir in dirmap.keys():
gtype = dirmap[gdir]
gatesdir = gdir
for f in os.listdir(gatesdir):
thegate = '/'.join([gatesdir, f])
if os.access(thegate, os.R_OK ^ os.X_OK):
tmpdir = anchore_utils.make_anchoretmpdir(self.config['tmpdir'])
namedir = '/'.join([tmpdir, 'namedir'])
os.makedirs(namedir)
imgfile = '/'.join([tmpdir, 'images'])
anchore_utils.write_plainfile_fromlist(imgfile, self.images)
cmdobj = scripting.ScriptExecutor(path=gatesdir, script_name=f)
out = cmdobj.execute(capture_output=True, cmdline=' '.join([imgfile,self.config['image_data_store'], namedir, 'anchore_getname']))
for ff in os.listdir(namedir):
if gtype not in ret:
ret[gtype] = {}
ret[gtype][f] = ff
shutil.rmtree(tmpdir)
return(ret)
def edit_policy_file(self, editpolicy=False, whitelist=False):
ret = True
if not editpolicy and not whitelist:
# nothing to do
return (ret)
for imageId in self.images:
if editpolicy:
data = self.anchoreDB.load_gate_policy(imageId)
else:
data = self.anchoreDB.load_gate_whitelist(imageId)
if not data:
self._logger.info(
"Cannot find existing data to edit, skipping: " +
str(imageId))
else:
tmpdir = anchore_utils.make_anchoretmpdir("/tmp")
try:
thefile = os.path.join(tmpdir, "anchorepol." + imageId)
anchore_utils.write_plainfile_fromlist(thefile, data)
if "EDITOR" in os.environ:
cmd = os.environ["EDITOR"].split()
cmd.append(thefile)
try:
subprocess.check_output(cmd, shell=False)
except:
ret = False
elif os.path.exists("/bin/vi"):
try:
rc = os.system("/bin/vi " + thefile)
if rc:
ret = False
except:
ret = False
else:
self._logger.info(
"Cannot find editor to use: please set the EDITOR environment variable and try again"
)
break
ret = False
newdata = anchore_utils.read_plainfile_tolist(thefile)
if editpolicy:
self.anchoreDB.save_gate_policy(imageId, newdata)
else:
self.anchoreDB.save_gate_whitelist(imageId, newdata)
except Exception as err:
pass
finally:
if tmpdir:
shutil.rmtree(tmpdir)
return (ret)
def save_gate_whitelist(self, imageId, data):
thedir = os.path.join(self.imagerootdir, imageId)
if not os.path.exists(thedir):
os.makedirs(thedir)
thefile = os.path.join(thedir, 'anchore_gate.whitelist')
return (anchore_utils.write_plainfile_fromlist(thefile, data))
def save_gate_eval_output(self, imageId, gate_name, data):
thedir = os.path.join(self.imagerootdir, imageId, "gates_output")
if not os.path.exists(thedir):
os.makedirs(thedir)
thefile = os.path.join(thedir, gate_name + ".eval")
return (anchore_utils.write_plainfile_fromlist(thefile, data))
def save_gate_whitelist(self, imageId, data):
thedir = os.path.join(self.imagerootdir, imageId)
if not os.path.exists(thedir):
os.makedirs(thedir)
thefile = os.path.join(thedir, 'anchore_gate.whitelist')
return(anchore_utils.write_plainfile_fromlist(thefile, data))
def save_gate_eval_output(self, imageId, gate_name, data):
thedir = os.path.join(self.imagerootdir, imageId, "gates_output")
if not os.path.exists(thedir):
os.makedirs(thedir)
thefile = os.path.join(thedir, gate_name+".eval")
return(anchore_utils.write_plainfile_fromlist(thefile, data))
def execute_query(self, imglist, se, params):
success = True
datadir = self.config['image_data_store']
outputdir = '/'.join([
self.config['anchore_data_dir'], "querytmp",
"query." + str(random.randint(0, 99999999))
])
if not os.path.exists(outputdir):
os.makedirs(outputdir)
imgfile = '/'.join([
self.config['anchore_data_dir'], "querytmp",
"queryimages." + str(random.randint(0, 99999999))
])
anchore_utils.write_plainfile_fromlist(imgfile, imglist)
cmdline = ' '.join([imgfile, datadir, outputdir])
if params:
cmdline = cmdline + ' ' + ' '.join(params)
meta = {}
try:
(cmd, rc, sout) = se.execute(capture_output=True, cmdline=cmdline)
if rc:
self._logger.error("Query command ran but execution failed")
self._logger.error("Query command: (" +
' '.join([se.thecmd, cmdline]) + ")")
self._logger.error("Query output: (" + str(sout) + ")")
self._logger.error("Exit code: (" + str(rc) + ")")
raise Exception("Query ran but exited non-zero.")
except Exception as err:
raise Exception("Query execution failed: " + str(err))
else:
try:
#outputs = os.listdir(outputdir)
warnfile = False
found = False
for f in os.listdir(outputdir):
if re.match(".*\.WARNS", f):
warnfile = '/'.join([outputdir, f])
else:
ofile = '/'.join([outputdir, f])
found = True
if not found:
raise Exception(
"No output files found after executing query command\n\tCommand Output:\n"
+ sout +
"\n\tInfo: Query command should have produced an output file in: "
+ outputdir)
orows = list()
try:
frows = anchore_utils.read_kvfile_tolist(ofile)
header = frows[0]
rowlen = len(header)
for row in frows[1:]:
if len(row) != rowlen:
raise Exception(
"Number of columns in data row (" +
str(len(row)) +
") is not equal to number of columns in header ("
+ str(rowlen) + ")\n\tHeader: " + str(header) +
"\n\tOffending Row: " + str(row))
orows.append(row)
except Exception as err:
raise err
if warnfile:
try:
meta['warns'] = anchore_utils.read_plainfile_tolist(
warnfile)
except:
pass
meta['queryparams'] = ','.join(params)
meta['querycommand'] = cmd
try:
i = header.index('URL')
meta['url_column_index'] = i
except:
pass
meta['result'] = {}
meta['result']['header'] = header
meta['result']['rowcount'] = len(orows)
try:
#meta['result']['colcount'] = len(orows[0])
meta['result']['colcount'] = len(header)
except:
meta['result']['colcount'] = 0
meta['result']['rows'] = orows
except Exception as err:
self._logger.error("Query output handling failed: ")
self._logger.error("\tCommand: " + str(cmd))
self._logger.error("\tException: " + str(err))
success = False
finally:
if imgfile and os.path.exists(imgfile):
os.remove(imgfile)
if outputdir and os.path.exists(outputdir):
shutil.rmtree(outputdir)
ret = [success, cmd, meta]
return (ret)
def execute_query(self, imglist, qcommand, params):
success = True
datadir = self.config['image_data_store']
outputdir = '/'.join([
self.config['anchore_data_dir'], "querytmp",
"query." + str(random.randint(0, 99999999))
])
os.makedirs(outputdir)
imgfile = '/'.join([
self.config['anchore_data_dir'], "querytmp",
"queryimages." + str(random.randint(0, 99999999))
])
anchore_utils.write_plainfile_fromlist(imgfile, imglist)
cmd = [qcommand, imgfile, datadir, outputdir]
if params:
cmd = cmd + params
meta = {}
try:
self._logger.debug("Running query command: " + str(' '.join(cmd)))
sout = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self._logger.debug("Query command execution success")
if sout:
self._logger.debug("\tCommand output:\n" + str(sout))
except subprocess.CalledProcessError as err:
self._logger.error("Query command ran but execution failed: ")
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
self._logger.error("\tCommand output:\n" + str(err.output))
self._logger.error("\tExit code: " + str(err.returncode))
success = False
except Exception as err:
self._logger.error("Query command execution call failed: ")
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
success = False
else:
try:
outputs = os.listdir(outputdir)
if len(outputs) <= 0:
raise Exception(
"No output files found after executing query command\n\tCommand Output:\n"
+ sout +
"\n\tInfo: Query command should have produced an output file in: "
+ outputdir)
orows = list()
ofile = outputs[0]
try:
frows = anchore_utils.read_kvfile_tolist('/'.join(
[outputdir, ofile]))
header = frows[0]
rowlen = len(header)
for row in frows[1:]:
if len(row) != rowlen:
raise Exception(
"Number of columns in data row (" +
str(len(row)) +
") is not equal to number of columns in header ("
+ str(rowlen) + ")\n\tHeader: " + str(header) +
"\n\tOffending Row: " + str(row))
orows.append(row)
except Exception as err:
raise err
meta['queryparams'] = ','.join(params)
meta['querycommand'] = cmd
meta['result'] = {}
meta['result']['header'] = header
meta['result']['rowcount'] = len(orows)
try:
meta['result']['colcount'] = len(orows[0])
except:
meta['result']['colcount'] = 0
meta['result']['rows'] = orows
except Exception as err:
self._logger.error("Query output handling failed: ")
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
success = False
finally:
os.remove(imgfile)
ret = [success, cmd, outputdir, meta]
return (ret)
def execute_query(self, imglist, qcommand, params):
success = True
datadir = self.config['image_data_store']
outputdir = '/'.join([self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999))])
os.makedirs(outputdir)
imgfile = '/'.join([self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999))])
anchore_utils.write_plainfile_fromlist(imgfile, imglist)
cmd = [qcommand, imgfile, datadir, outputdir]
if params:
cmd = cmd + params
meta = {}
try:
self._logger.debug("Running query command: " + str(' '.join(cmd)))
sout = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self._logger.debug("Query command execution success")
if sout:
self._logger.debug("\tCommand output:\n" + str(sout))
except subprocess.CalledProcessError as err:
self._logger.error("Query command ran but execution failed: " )
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
self._logger.error("\tCommand output:\n" + str(err.output))
self._logger.error("\tExit code: " + str(err.returncode))
success = False
except Exception as err:
self._logger.error("Query command execution call failed: " )
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
success = False
else:
try:
outputs = os.listdir(outputdir)
if len(outputs) <= 0:
raise Exception("No output files found after executing query command\n\tCommand Output:\n"+sout+"\n\tInfo: Query command should have produced an output file in: " + outputdir)
orows = list()
ofile = outputs[0]
try:
frows = anchore_utils.read_kvfile_tolist('/'.join([outputdir, ofile]))
header = frows[0]
rowlen = len(header)
for row in frows[1:]:
if len(row) != rowlen:
raise Exception("Number of columns in data row ("+str(len(row))+") is not equal to number of columns in header ("+str(rowlen)+")\n\tHeader: "+str(header)+"\n\tOffending Row: "+str(row))
orows.append(row)
except Exception as err:
raise err
meta['queryparams'] = ','.join(params)
meta['querycommand'] = cmd
meta['result'] = {}
meta['result']['header'] = header
meta['result']['rowcount'] = len(orows)
try:
meta['result']['colcount'] = len(orows[0])
except:
meta['result']['colcount'] = 0
meta['result']['rows'] = orows
except Exception as err:
self._logger.error("Query output handling failed: ")
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
success = False
finally:
os.remove(imgfile)
ret = [success, cmd, outputdir, meta]
return(ret)
def execute_query(self, imglist, se, params):
success = True
datadir = self.config['image_data_store']
outputdir = '/'.join([self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999))])
if not os.path.exists(outputdir):
os.makedirs(outputdir)
imgfile = '/'.join([self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999))])
anchore_utils.write_plainfile_fromlist(imgfile, imglist)
cmdline = ' '.join([imgfile, datadir, outputdir])
if params:
cmdline = cmdline + ' ' + ' '.join(params)
meta = {}
try:
(cmd, rc, sout) = se.execute(capture_output=True, cmdline=cmdline)
if rc:
self._logger.error("Query command ran but execution failed" )
self._logger.error("Query command: (" + ' '.join([se.thecmd, cmdline])+")")
self._logger.error("Query output: (" + str(sout) + ")")
self._logger.error("Exit code: (" + str(rc)+")")
raise Exception("Query ran but exited non-zero.")
except Exception as err:
raise Exception("Query execution failed: " + str(err))
else:
try:
#outputs = os.listdir(outputdir)
warnfile = False
found = False
for f in os.listdir(outputdir):
if re.match(".*\.WARNS", f):
warnfile = '/'.join([outputdir, f])
else:
ofile = '/'.join([outputdir, f])
found=True
if not found:
raise Exception("No output files found after executing query command\n\tCommand Output:\n"+sout+"\n\tInfo: Query command should have produced an output file in: " + outputdir)
orows = list()
try:
frows = anchore_utils.read_kvfile_tolist(ofile)
header = frows[0]
rowlen = len(header)
for row in frows[1:]:
if len(row) != rowlen:
raise Exception("Number of columns in data row ("+str(len(row))+") is not equal to number of columns in header ("+str(rowlen)+")\n\tHeader: "+str(header)+"\n\tOffending Row: "+str(row))
orows.append(row)
except Exception as err:
raise err
if warnfile:
try:
meta['warns'] = anchore_utils.read_plainfile_tolist(warnfile)
except:
pass
meta['queryparams'] = ','.join(params)
meta['querycommand'] = cmd
try:
i = header.index('URL')
meta['url_column_index'] = i
except:
pass
meta['result'] = {}
meta['result']['header'] = header
meta['result']['rowcount'] = len(orows)
try:
#meta['result']['colcount'] = len(orows[0])
meta['result']['colcount'] = len(header)
except:
meta['result']['colcount'] = 0
meta['result']['rows'] = orows
except Exception as err:
self._logger.error("Query output handling failed: ")
self._logger.error("\tCommand: " + str(cmd))
self._logger.error("\tException: " + str(err))
success = False
finally:
if imgfile and os.path.exists(imgfile):
os.remove(imgfile)
if outputdir and os.path.exists(outputdir):
shutil.rmtree(outputdir)
ret = [success, cmd, meta]
return(ret)
