def build_image_list(config, image, imagefile, all_local, include_allanchore, dockerfile=None, exclude_file=None):
"""Given option inputs from the cli, construct a list of image ids. Includes all found with no exclusion logic"""
if not image and not (imagefile or all_local):
raise click.BadOptionUsage('No input found for image source. One of <image>, <imagefile>, or <all> must be specified')
if image and imagefile:
raise click.BadOptionUsage('Only one of <image> and <imagefile> can be specified')
filter_images = []
if exclude_file:
with open(exclude_file) as f:
for line in f.readlines():
filter_images.append(line.strip())
imagelist = {}
if image:
imagelist[image] = {'dockerfile':dockerfile}
if imagefile:
filelist = anchore_utils.read_kvfile_tolist(imagefile)
for i in range(len(filelist)):
l = filelist[i]
imageId = l[0]
try:
dfile = l[1]
except:
dfile = None
imagelist[imageId] = {'dockerfile':dfile}
if all_local:
docker_cli = contexts['docker_cli']
if docker_cli:
for f in docker_cli.images(all=True, quiet=True, filters={'dangling': False}):
if f not in imagelist and f not in filter_images:
imagelist[f] = {'dockerfile':None}
else:
raise Exception("Could not load any images from local docker host - is docker running?")
if include_allanchore:
ret = contexts['anchore_db'].load_all_images()
if ret and len(ret) > 0:
for l in list(set(imagelist.keys()) | set(ret)):
imagelist[l] = {'dockerfile':None}
# Remove excluded items
for excluded in filter_images:
docker_cli = contexts['docker_cli']
if not docker_cli:
raise Exception("Could not query docker - is docker running?")
for img in docker_cli.images(name=excluded, quiet=True):
imagelist.pop(img, None)
#imagelist.remove(img)
return imagelist
def build_image_list(config, image, imagefile, all_local, include_allanchore, dockerfile=None, exclude_file=None):
"""Given option inputs from the cli, construct a list of image ids. Includes all found with no exclusion logic"""
if not image and not (imagefile or all_local):
raise click.BadOptionUsage('No input found for image source. One of <image>, <imagefile>, or <all> must be specified')
if image and imagefile:
raise click.BadOptionUsage('Only one of <image> and <imagefile> can be specified')
filter_images = []
if exclude_file:
with open(exclude_file) as f:
for line in f.readlines():
filter_images.append(line.strip())
imagelist = {}
if image:
imagelist[image] = {'dockerfile':dockerfile}
if imagefile:
filelist = anchore_utils.read_kvfile_tolist(imagefile)
for i in range(len(filelist)):
l = filelist[i]
imageId = l[0]
try:
dfile = l[1]
except:
dfile = None
imagelist[imageId] = {'dockerfile':dfile}
if all_local:
docker_cli = contexts['docker_cli']
if docker_cli:
for f in docker_cli.images(all=True, quiet=True, filters={'dangling': False}):
if f not in imagelist and f not in filter_images:
imagelist[f] = {'dockerfile':None}
else:
raise Exception("Could not load any images from local docker host - is docker running?")
if include_allanchore:
ret = list(contexts['anchore_db'].load_all_images().keys())
if ret and len(ret) > 0:
for l in list(set(imagelist.keys()) | set(ret)):
imagelist[l] = {'dockerfile':None}
# Remove excluded items
for excluded in filter_images:
docker_cli = contexts['docker_cli']
if not docker_cli:
raise Exception("Could not query docker - is docker running?")
for img in docker_cli.images(name=excluded, quiet=True):
imagelist.pop(img, None)
#imagelist.remove(img)
return imagelist
def load_global_whitelist(self):
ret = []
whitelist_data = []
whitelist_file = None
if self.global_whitelist_override and os.path.exists(self.global_whitelist_override):
whitelist_file = self.global_whitelist_override
elif self.default_global_whitelist and os.path.exists(self.default_global_whitelist):
whitelist_file = self.default_global_whitelist
else:
self._logger.debug("no global whitelist can be found, skipping")
if whitelist_file:
whitelist_data = anchore_utils.read_kvfile_tolist(whitelist_file)
for item in whitelist_data:
if item[0] and not re.match("^#", item[0]) and len(item) > 1:
store = item[0:2]
ret.append(store)
return(ret)
def execute_query(self, imglist, se, params):
success = True
datadir = self.config['image_data_store']
outputdir = '/'.join([
self.config['anchore_data_dir'], "querytmp",
"query." + str(random.randint(0, 99999999))
])
if not os.path.exists(outputdir):
os.makedirs(outputdir)
imgfile = '/'.join([
self.config['anchore_data_dir'], "querytmp",
"queryimages." + str(random.randint(0, 99999999))
])
anchore_utils.write_plainfile_fromlist(imgfile, imglist)
cmdline = ' '.join([imgfile, datadir, outputdir])
if params:
cmdline = cmdline + ' ' + ' '.join(params)
meta = {}
try:
(cmd, rc, sout) = se.execute(capture_output=True, cmdline=cmdline)
if rc:
self._logger.error("Query command ran but execution failed")
self._logger.error("Query command: (" +
' '.join([se.thecmd, cmdline]) + ")")
self._logger.error("Query output: (" + str(sout) + ")")
self._logger.error("Exit code: (" + str(rc) + ")")
raise Exception("Query ran but exited non-zero.")
except Exception as err:
raise Exception("Query execution failed: " + str(err))
else:
try:
#outputs = os.listdir(outputdir)
warnfile = False
found = False
for f in os.listdir(outputdir):
if re.match(".*\.WARNS", f):
warnfile = '/'.join([outputdir, f])
else:
ofile = '/'.join([outputdir, f])
found = True
if not found:
raise Exception(
"No output files found after executing query command\n\tCommand Output:\n"
+ sout +
"\n\tInfo: Query command should have produced an output file in: "
+ outputdir)
orows = list()
try:
frows = anchore_utils.read_kvfile_tolist(ofile)
header = frows[0]
rowlen = len(header)
for row in frows[1:]:
if len(row) != rowlen:
raise Exception(
"Number of columns in data row (" +
str(len(row)) +
") is not equal to number of columns in header ("
+ str(rowlen) + ")\n\tHeader: " + str(header) +
"\n\tOffending Row: " + str(row))
orows.append(row)
except Exception as err:
raise err
if warnfile:
try:
meta['warns'] = anchore_utils.read_plainfile_tolist(
warnfile)
except:
pass
meta['queryparams'] = ','.join(params)
meta['querycommand'] = cmd
try:
i = header.index('URL')
meta['url_column_index'] = i
except:
pass
meta['result'] = {}
meta['result']['header'] = header
meta['result']['rowcount'] = len(orows)
try:
#meta['result']['colcount'] = len(orows[0])
meta['result']['colcount'] = len(header)
except:
meta['result']['colcount'] = 0
meta['result']['rows'] = orows
except Exception as err:
self._logger.error("Query output handling failed: ")
self._logger.error("\tCommand: " + str(cmd))
self._logger.error("\tException: " + str(err))
success = False
finally:
if imgfile and os.path.exists(imgfile):
os.remove(imgfile)
if outputdir and os.path.exists(outputdir):
shutil.rmtree(outputdir)
ret = [success, cmd, meta]
return (ret)
def execute_query(self, imglist, qcommand, params):
success = True
datadir = self.config['image_data_store']
outputdir = '/'.join([
self.config['anchore_data_dir'], "querytmp",
"query." + str(random.randint(0, 99999999))
])
os.makedirs(outputdir)
imgfile = '/'.join([
self.config['anchore_data_dir'], "querytmp",
"queryimages." + str(random.randint(0, 99999999))
])
anchore_utils.write_plainfile_fromlist(imgfile, imglist)
cmd = [qcommand, imgfile, datadir, outputdir]
if params:
cmd = cmd + params
meta = {}
try:
self._logger.debug("Running query command: " + str(' '.join(cmd)))
sout = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self._logger.debug("Query command execution success")
if sout:
self._logger.debug("\tCommand output:\n" + str(sout))
except subprocess.CalledProcessError as err:
self._logger.error("Query command ran but execution failed: ")
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
self._logger.error("\tCommand output:\n" + str(err.output))
self._logger.error("\tExit code: " + str(err.returncode))
success = False
except Exception as err:
self._logger.error("Query command execution call failed: ")
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
success = False
else:
try:
outputs = os.listdir(outputdir)
if len(outputs) <= 0:
raise Exception(
"No output files found after executing query command\n\tCommand Output:\n"
+ sout +
"\n\tInfo: Query command should have produced an output file in: "
+ outputdir)
orows = list()
ofile = outputs[0]
try:
frows = anchore_utils.read_kvfile_tolist('/'.join(
[outputdir, ofile]))
header = frows[0]
rowlen = len(header)
for row in frows[1:]:
if len(row) != rowlen:
raise Exception(
"Number of columns in data row (" +
str(len(row)) +
") is not equal to number of columns in header ("
+ str(rowlen) + ")\n\tHeader: " + str(header) +
"\n\tOffending Row: " + str(row))
orows.append(row)
except Exception as err:
raise err
meta['queryparams'] = ','.join(params)
meta['querycommand'] = cmd
meta['result'] = {}
meta['result']['header'] = header
meta['result']['rowcount'] = len(orows)
try:
meta['result']['colcount'] = len(orows[0])
except:
meta['result']['colcount'] = 0
meta['result']['rows'] = orows
except Exception as err:
self._logger.error("Query output handling failed: ")
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
success = False
finally:
os.remove(imgfile)
ret = [success, cmd, outputdir, meta]
return (ret)
def execute_query(self, imglist, qcommand, params):
success = True
datadir = self.config['image_data_store']
outputdir = '/'.join([self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999))])
os.makedirs(outputdir)
imgfile = '/'.join([self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999))])
anchore_utils.write_plainfile_fromlist(imgfile, imglist)
cmd = [qcommand, imgfile, datadir, outputdir]
if params:
cmd = cmd + params
meta = {}
try:
self._logger.debug("Running query command: " + str(' '.join(cmd)))
sout = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self._logger.debug("Query command execution success")
if sout:
self._logger.debug("\tCommand output:\n" + str(sout))
except subprocess.CalledProcessError as err:
self._logger.error("Query command ran but execution failed: " )
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
self._logger.error("\tCommand output:\n" + str(err.output))
self._logger.error("\tExit code: " + str(err.returncode))
success = False
except Exception as err:
self._logger.error("Query command execution call failed: " )
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
success = False
else:
try:
outputs = os.listdir(outputdir)
if len(outputs) <= 0:
raise Exception("No output files found after executing query command\n\tCommand Output:\n"+sout+"\n\tInfo: Query command should have produced an output file in: " + outputdir)
orows = list()
ofile = outputs[0]
try:
frows = anchore_utils.read_kvfile_tolist('/'.join([outputdir, ofile]))
header = frows[0]
rowlen = len(header)
for row in frows[1:]:
if len(row) != rowlen:
raise Exception("Number of columns in data row ("+str(len(row))+") is not equal to number of columns in header ("+str(rowlen)+")\n\tHeader: "+str(header)+"\n\tOffending Row: "+str(row))
orows.append(row)
except Exception as err:
raise err
meta['queryparams'] = ','.join(params)
meta['querycommand'] = cmd
meta['result'] = {}
meta['result']['header'] = header
meta['result']['rowcount'] = len(orows)
try:
meta['result']['colcount'] = len(orows[0])
except:
meta['result']['colcount'] = 0
meta['result']['rows'] = orows
except Exception as err:
self._logger.error("Query output handling failed: ")
self._logger.error("\tCommand: " + ' '.join(cmd))
self._logger.error("\tException: " + str(err))
success = False
finally:
os.remove(imgfile)
ret = [success, cmd, outputdir, meta]
return(ret)
def execute_query(self, imglist, se, params):
success = True
datadir = self.config['image_data_store']
outputdir = '/'.join([self.config['anchore_data_dir'], "querytmp", "query." + str(random.randint(0, 99999999))])
if not os.path.exists(outputdir):
os.makedirs(outputdir)
imgfile = '/'.join([self.config['anchore_data_dir'], "querytmp", "queryimages." + str(random.randint(0, 99999999))])
anchore_utils.write_plainfile_fromlist(imgfile, imglist)
cmdline = ' '.join([imgfile, datadir, outputdir])
if params:
cmdline = cmdline + ' ' + ' '.join(params)
meta = {}
try:
(cmd, rc, sout) = se.execute(capture_output=True, cmdline=cmdline)
if rc:
self._logger.error("Query command ran but execution failed" )
self._logger.error("Query command: (" + ' '.join([se.thecmd, cmdline])+")")
self._logger.error("Query output: (" + str(sout) + ")")
self._logger.error("Exit code: (" + str(rc)+")")
raise Exception("Query ran but exited non-zero.")
except Exception as err:
raise Exception("Query execution failed: " + str(err))
else:
try:
#outputs = os.listdir(outputdir)
warnfile = False
found = False
for f in os.listdir(outputdir):
if re.match(".*\.WARNS", f):
warnfile = '/'.join([outputdir, f])
else:
ofile = '/'.join([outputdir, f])
found=True
if not found:
raise Exception("No output files found after executing query command\n\tCommand Output:\n"+sout+"\n\tInfo: Query command should have produced an output file in: " + outputdir)
orows = list()
try:
frows = anchore_utils.read_kvfile_tolist(ofile)
header = frows[0]
rowlen = len(header)
for row in frows[1:]:
if len(row) != rowlen:
raise Exception("Number of columns in data row ("+str(len(row))+") is not equal to number of columns in header ("+str(rowlen)+")\n\tHeader: "+str(header)+"\n\tOffending Row: "+str(row))
orows.append(row)
except Exception as err:
raise err
if warnfile:
try:
meta['warns'] = anchore_utils.read_plainfile_tolist(warnfile)
except:
pass
meta['queryparams'] = ','.join(params)
meta['querycommand'] = cmd
try:
i = header.index('URL')
meta['url_column_index'] = i
except:
pass
meta['result'] = {}
meta['result']['header'] = header
meta['result']['rowcount'] = len(orows)
try:
#meta['result']['colcount'] = len(orows[0])
meta['result']['colcount'] = len(header)
except:
meta['result']['colcount'] = 0
meta['result']['rows'] = orows
except Exception as err:
self._logger.error("Query output handling failed: ")
self._logger.error("\tCommand: " + str(cmd))
self._logger.error("\tException: " + str(err))
success = False
finally:
if imgfile and os.path.exists(imgfile):
os.remove(imgfile)
if outputdir and os.path.exists(outputdir):
shutil.rmtree(outputdir)
ret = [success, cmd, meta]
return(ret)