def create_definition(self, cve_id, summary, test_ref_path, new_def_path):
self.cve_id = cve_id
self.summary = summary
self.test_ref_id = lib_repo.path_to_oval_id(test_ref_path)
self.new_def_path = new_def_path
self.def_id = lib_repo.path_to_oval_id(new_def_path)
self.def_platform = 'GenericPlatform'
self.criteria_comment = 'Automatically added for NVD CVE {0}'.format(self.cve_id)
self.def_family = 'Linux'
self.def_cve_ref_url = mitre_cve_url + self.cve_id
self.def_criterion_comment = 'Automatically generated for NVD CVE {0}'.format(self.cve_id)
self.date = date.today().strftime('%Y-%m-%dT%H:%M:%S%z')
doc_root = etree.fromstring(oval_definition)
for node in doc_root.getiterator():
if self.namespace + 'definition' in node.tag:
node.set('id', self.def_id)
if self.namespace + 'title' in node.tag:
node.text = self.summary
if self.namespace + 'affected' in node.tag:
node.set('family', self.def_family)
if self.namespace + 'platform' in node.tag:
node.text = self.def_platform
if self.namespace + 'reference' in node.tag:
node.set('ref_id', self.cve_id)
node.set('ref_url', self.def_cve_ref_url)
if self.namespace + 'description' in node.tag:
node.text = self.summary
if self.namespace + 'submitted' in node.tag:
node.set('date', str(self.date))
if self.namespace + 'contributor' in node.tag:
node.set('organization', 'Automatically generated from NVD')
node.text = 'Automatically generated from NVD'
if self.namespace + 'status_change' in node.tag:
node.set('date', str(self.date))
node.text = 'ACCEPTED'
if self.namespace + 'status' in node.tag:
node.text = 'ACCEPTED'
if self.namespace + 'criterion' in node.tag:
node.set('comment', 'Automatically added for NVD CVE {0}'.format(self.cve_id))
node.set('test_ref', self.test_ref_id)
NVDOVALXMLWriter().write(self.new_def_path, doc_root)
return
def put(self, cve_id, test_path, object_path, state_path):
self.test_path, self.object_path, self.state_path = test_path, object_path, state_path
self.check = 'all'
self.check_existence = 'at_least_one_exists'
self.comment = 'Automatically added for NVD CVE {0}'.format(cve_id)
self.test_id = lib_repo.path_to_oval_id(self.test_path)
self.object_id = lib_repo.path_to_oval_id(self.object_path)
self.state_id = lib_repo.path_to_oval_id(self.state_path)
self.make()
return
def put(self, cve_id, test_path, object_path, state_path):
self.test_path, self.object_path, self.state_path = test_path, object_path, state_path
self.check = 'all'
self.check_existence = 'at_least_one_exists'
self.comment = 'Automatically added for NVD CVE {0}'.format(cve_id)
self.test_id = lib_repo.path_to_oval_id(self.test_path)
self.object_id = lib_repo.path_to_oval_id(self.object_path)
self.state_id = lib_repo.path_to_oval_id(self.state_path)
self.make()
return
def main():
orphans = {}
elements_index = lib_search.ElementsIndex(message)
for elempath in lib_repo.get_element_paths_iterator():
oval_id = lib_repo.path_to_oval_id(elempath)
oval_ids = { oval_id }
upstream_ids = elements_index.find_upstream_ids(oval_ids, set())
downstream_ids = elements_index.find_downstream_ids(oval_ids, set())
#message("INFO", "OVAL ID '%s' had %i upstream ids and %i downstream ids" % (oval_id, len(upstream_ids), len(downstream_ids)))
if len(upstream_ids) == 0 and len(downstream_ids) == 0:
message("INFO", "Found Orphan '%s'" % oval_id)
orphans[oval_id] = elempath
response = input("\n :::: Remove all orphans? (N[o] / y[es]): ")
if response != 'y' and response != 'Y':
return
message("INFO", "\n\n")
for ok in orphans.keys():
message("INFO", "Deleting Orphan '%s'" % ok)
os.remove(orphans[ok])
def main():
start_time = time.time()
definitions_index = lib_search.DefinitionsIndex(message)
ns_map = {"oval-def": "http://oval.mitre.org/XMLSchema/oval-definitions-5"}
# Create the output object...
output = WebOutput()
for defpath in lib_repo.get_definition_paths_iterator():
def_id = lib_repo.path_to_oval_id(defpath)
try:
tree = etree.parse(defpath)
except OSError as e:
raise InvalidPathError(defpath)
except etree.XMLSyntaxError as e:
raise InvalidXmlError(defpath, e.msg)
root = tree.getroot()
# Only Vulnerability and Patch definitions would have CVE references...
defclass = root.get("class")
if defclass in ["vulnerability", "patch"]:
reference = root.find("./oval-def:metadata/oval-def:reference[@source='CVE']", ns_map)
if reference is not None:
output.add_result(def_id, [reference.get("ref_id"), reference.get("ref_url")])
# add timing
seconds_elapsed = time.time() - start_time
output.message("CVE", "OVAL Repository Definition-to-CVE Mappings")
output.message("time", "{0}".format(format_duration(seconds_elapsed)))
# Write JSON Output...
output.write_json()
def determine_def_min_version(defpath, definitions_index, elements_index, update):
""" determines the minimum oval schema version the given definition can be expressed in. """
global schema_path_cache
# build schema path cache
for schema_version in lib_repo.get_schema_versions():
schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)
minimum_version=None
exception=None
# get id of oval definition
def_id = lib_repo.path_to_oval_id(defpath)
# add all downstream element ids
def_ids = set([def_id])
oval_ids = elements_index.find_downstream_ids(def_ids, def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
# create generator that builds files in memory (these are small)
OvalGenerator = lib_xml.OvalGenerator(message)
OvalGenerator.use_file_queues = False
# add each OVAL definition to generator
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
# parse defintion, get ref to schema_version element
xmlstring = OvalGenerator.to_string(True, False)
xmlstring = xmlstring.replace('<?xml version="1.0" encoding="UTF-8"?>','')
xmlstring = xmlstring.replace('<?xml version="1.0" encoding="utf-8"?>','')
xmlstring = xmlstring.replace("<?xml version='1.0' encoding='UTF-8'?>",'')
xmlstring = xmlstring.replace("<?xml version='1.0' encoding='utf-8'?>",'')
#message("INFO", "XMLSTRING is %s" % xmlstring)
#tree = etree.fromstring(OvalGenerator.to_string(True, False))
tree = etree.XML(xmlstring)
schema_version_element = tree.find('.//oval:schema_version', { 'oval': 'http://oval.mitre.org/XMLSchema/oval-common-5' })
for schema_version in lib_repo.get_schema_versions():
# update schema version in tree
schema_version_element.text = schema_version
# test of definitions file validates against current schema
try:
message("INFO", "Schema Path is %s" % schema_path_cache[schema_version])
lib_xml.schema_validate(tree, schema_path_cache[schema_version])
minimum_version = schema_version
message("INFO", "XML is valid against schema version %s" % schema_version)
except lib_xml.SchemaValidationError as e:
exception = e.message
message("ERROR", "Exception (schema version is %s) - %s" % (schema_version, e.message))
break;
if update:
set_min_schema_version(defpath, minimum_version)
return { 'oval_id': def_id, 'minimum_version': minimum_version, 'exception': exception }
def put(self, cve_id, xml_path, product):
self.cve_id, self.xml_path, self.product = cve_id, xml_path, product
self.comment = 'Automatically added for NVD CVE {0}'.format(cve_id)
self.this_id = lib_repo.path_to_oval_id(self.xml_path)
self.make()
return
def put(self, cve_id, xml_path, product):
self.cve_id, self.xml_path, self.product = cve_id, xml_path, product
self.comment = 'Automatically added for NVD CVE {0}'.format(cve_id)
self.this_id = lib_repo.path_to_oval_id(self.xml_path)
self.make()
return
def get_last_def(self):
for xml_file in lib_repo.get_element_paths_iterator():
if 'vulnerability' in xml_file:
current_id = int(lib_repo.path_to_oval_id(xml_file).split(':')[3])
if current_id > self.new_id:
self.new_id = current_id
self.path = os.path.split(xml_file)[0]
self.new_filename = os.path.split(xml_file)[1]
return
def get_last(self):
for xml_file in lib_repo.get_element_paths_iterator():
if (self.tst_or_obj_or_ste in xml_file) and (self.family in xml_file):
current_id = int(lib_repo.path_to_oval_id(xml_file).split(':')[3])
if current_id > self.new_id:
self.new_id = current_id
self.path = os.path.split(xml_file)[0]
self.new_filename = os.path.split(xml_file)[1]
return
def get_last_def(self):
for xml_file in lib_repo.get_element_paths_iterator():
if 'vulnerability' in xml_file:
current_id = int(
lib_repo.path_to_oval_id(xml_file).split(':')[3])
if current_id > self.new_id:
self.new_id = current_id
self.path = os.path.split(xml_file)[0]
self.new_filename = os.path.split(xml_file)[1]
return
def put(self, cve_id, summary, test_ref_path, new_def_path):
self.cve_id = cve_id
self.summary = summary
self.test_ref_id = lib_repo.path_to_oval_id(test_ref_path)
self.definition_id = lib_repo.path_to_oval_id(new_def_path)
self.platform = 'GenericPlatform'
self.criteria_comment = 'Automatically added for NVD CVE {0}'.format(self.cve_id)
# check it the OVAL repository contains the CVE
# if yes, update the definition with stub containing the definition created above
if self.get_definition_path():
self.update_definition()
else:
NVDOVALDefinitionVulnerabilityCreate().create_definition(cve_id, summary, test_ref_path, new_def_path)
# create new definition
# NVDOVALDefinitionVulnerabilityCreate().create_definition(cve_id, summary, test_ref_path, new_def_path)
return
def get_last(self):
for xml_file in lib_repo.get_element_paths_iterator():
if (self.tst_or_obj_or_ste in xml_file) and (self.family
in xml_file):
current_id = int(
lib_repo.path_to_oval_id(xml_file).split(':')[3])
if current_id > self.new_id:
self.new_id = current_id
self.path = os.path.split(xml_file)[0]
self.new_filename = os.path.split(xml_file)[1]
return
def build_comprehensive_oval_document(changes):
"""
Builds an XML tree which contains all elements affected by the changes
"""
global debug
global verbose
if changes is None or len(changes) < 1:
return None
if verbose:
print(" ---- Getting OVAL ID's for all changed files...")
oval_ids_changed = {
lib_repo.path_to_oval_id(filepath)
for filepath in changes
}
# find all upstream ids
if verbose:
print(
" ---- Locating parent definitions for all changed elements...")
elements_index = lib_search.ElementsIndex(False)
upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set())
# filter affected to definition ids
affected_def_ids = {
oval_id
for oval_id in upstream_ids
if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition'
}
# get all downstream elements
if verbose:
print(
" ---- Resolving all elements needed to build comprehensive document..."
)
oval_ids = elements_index.find_downstream_ids(affected_def_ids,
affected_def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
if verbose:
print(
" ---- Importing separate elements into comprehensive document...."
)
oval = OvalDocument(None)
for path in file_paths:
element = OvalElement.fromStandaloneFile(path)
if element is None:
print(":::: None from path: ", path)
return None
oval.addElement(element, True)
return etree.fromstring(oval.to_string())
def put(self, cve_id, xml_path, version):
self.cve_id, self.xml_path, self.version = cve_id, xml_path, version
self.comment = 'Automatically added for NVD CVE {0}'.format(cve_id)
self.this_id = lib_repo.path_to_oval_id(self.xml_path)
self.version = '0:' + self.version
self.datatype = 'evr_string'
self.operation = 'less than or equal'
self.make()
return
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(
description='Builds all OVAL definitons in the repository.')
parser.add_argument(
'-l',
'--limit',
nargs='?',
default="0",
type=int,
help='limits number of definitions that will be built)')
args = vars(parser.parse_args())
# get indexes
# definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
# create generator, build in memory b/c smallish files
OvalGenerator = lib_xml.OvalGenerator(message)
OvalGenerator.use_file_queues = False
print("Storing data in temp directory: {}".format(tempfile.gettempdir()))
i_file = 0
i_limit = args['limit']
for defpath in lib_repo.get_definition_paths_iterator():
i_file = i_file + 1
if i_limit > 0 and i_file > i_limit:
break
def_id = lib_repo.path_to_oval_id(defpath)
message('info', 'Building file {0} for {1}.'.format(i_file, def_id))
# add all downstream element ids
def_ids = set([def_id])
oval_ids = elements_index.find_downstream_ids(def_ids, def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
# add each OVAL definition to generator
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
# write output file
outfile = '{1}/gen.{0}'.format(lib_repo.oval_id_to_path(def_id),
tempfile.gettempdir())
OvalGenerator.to_file(outfile)
seconds_elapsed = time.time() - start_time
message('info',
'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def put(self, cve_id, xml_path, version):
self.cve_id, self.xml_path, self.version = cve_id, xml_path, version
self.comment = 'Automatically added for NVD CVE {0}'.format(cve_id)
self.this_id = lib_repo.path_to_oval_id(self.xml_path)
self.version = '0:' + self.version
self.datatype = 'evr_string'
self.operation = 'less than or equal'
self.make()
return
def determine_def_min_version(defpath, definitions_index, elements_index, update):
""" determines the minimum oval schema version the given definition can be expressed in. """
global schema_path_cache
# build schema path cache
for schema_version in lib_repo.get_schema_versions():
schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)
minimum_version=None
exception=None
# get id of oval definition
def_id = lib_repo.path_to_oval_id(defpath)
# add all downstream element ids
def_ids = set([def_id])
oval_ids = elements_index.find_downstream_ids(def_ids, def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
# create generator that builds files in memory (these are small)
OvalGenerator = lib_xml.OvalGenerator(message)
OvalGenerator.use_file_queues = False
# add each OVAL definition to generator
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
# parse defintion, get ref to schema_version element
tree = etree.fromstring(OvalGenerator.to_string(True, False))
schema_version_element = tree.find('.//oval:schema_version', { 'oval': 'http://oval.mitre.org/XMLSchema/oval-common-5' })
for schema_version in lib_repo.get_schema_versions():
# update schema version in tree
schema_version_element.text = schema_version
# test of definitions file validates against current schema
try:
message("INFO", "Schema Path is %s" % schema_path_cache[schema_version])
lib_xml.schema_validate(tree, schema_path_cache[schema_version])
minimum_version = schema_version
message("INFO", "XML is valid against schema version %s" % schema_version)
except lib_xml.SchemaValidationError as e:
exception = e.message
message("ERROR", "Exception (schema version is %s) - %s" % (schema_version, e.message))
break;
if update:
set_min_schema_version(defpath, minimum_version)
return { 'oval_id': def_id, 'minimum_version': minimum_version, 'exception': exception }
def put(self, cve_id, summary, test_ref_path, new_def_path):
self.cve_id = cve_id
self.summary = summary
self.test_ref_id = lib_repo.path_to_oval_id(test_ref_path)
self.definition_id = lib_repo.path_to_oval_id(new_def_path)
self.platform = 'GenericPlatform'
self.criteria_comment = 'Automatically added for NVD CVE {0}'.format(
self.cve_id)
# check it the OVAL repository contains the CVE
# if yes, update the definition with stub containing the definition created above
if self.get_definition_path():
self.update_definition()
else:
NVDOVALDefinitionVulnerabilityCreate().create_definition(
cve_id, summary, test_ref_path, new_def_path)
# create new definition
# NVDOVALDefinitionVulnerabilityCreate().create_definition(cve_id, summary, test_ref_path, new_def_path)
return
def get_changed_ids_from_git():
try:
changed_files = lib_git.get_uncommitted_oval()
if not changed_files or changed_files is None:
return None
change_list = []
for file in changed_files:
change_list.add(lib_repo.path_to_oval_id(file))
return change_list
except Exception:
if main.verbose:
print("## Error while querying git for changes: ", format(Exception))
return None
def determine_def_min_version(defpath, definitions_index, elements_index, update):
""" determines the minimum oval schema version the given definition can be expressed in. """
global schema_path_cache
minimum_version = None
exception = None
# get id of oval definition
def_id = lib_repo.path_to_oval_id(defpath)
# add all downstream element ids
def_ids = set([def_id])
oval_ids = elements_index.find_downstream_ids(def_ids, def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
# create generator that builds files in memory (these are small)
OvalGenerator = lib_xml.OvalGenerator(message)
OvalGenerator.use_file_queues = False
# add each OVAL definition to generator
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
# parse defintion, get ref to schema_version element
tree = etree.fromstring(OvalGenerator.to_string())
schema_version_element = tree.find(
".//oval:schema_version", {"oval": "http://oval.mitre.org/XMLSchema/oval-common-5"}
)
for schema_version in lib_repo.get_schema_versions():
# update schema version in tree
schema_version_element.text = schema_version
# test of definitions file validates against current schema
try:
lib_xml.schema_validate(tree, schema_path_cache[schema_version])
minimum_version = schema_version
except lib_xml.SchemaValidationError as e:
exception = e.message
break
if update:
set_min_schema_version(defpath, minimum_version)
return {"oval_id": def_id, "minimum_version": minimum_version, "exception": exception}
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(description="Builds all OVAL definitons in the repository.")
parser.add_argument(
"-l", "--limit", nargs="?", default="0", type=int, help="limits number of definitions that will be built)"
)
args = vars(parser.parse_args())
# get indexes
definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
# create generator, build in memory b/c smallish files
OvalGenerator = lib_xml.OvalGenerator(message)
OvalGenerator.use_file_queues = False
i_file = 0
i_limit = args["limit"]
for defpath in lib_repo.get_definition_paths_iterator():
i_file = i_file + 1
if i_limit > 0 and i_file > i_limit:
break
def_id = lib_repo.path_to_oval_id(defpath)
message("info", "Building file {0} for {1}.".format(i_file, def_id))
# add all downstream element ids
def_ids = set([def_id])
oval_ids = elements_index.find_downstream_ids(def_ids, def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
# add each OVAL definition to generator
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
# write output file
outfile = "./tmp/gen.{0}".format(lib_repo.oval_id_to_path(def_id))
OvalGenerator.to_file(outfile)
seconds_elapsed = time.time() - start_time
message("info", "Completed in {0}!".format(format_duration(seconds_elapsed)))
def build_comprehensive_oval_document(changes):
"""
Builds an XML tree which contains all elements affected by the changes
"""
global debug
global verbose
if changes is None or len(changes) < 1:
return None
if verbose:
print(" ---- Getting OVAL ID's for all changed files...")
oval_ids_changed = {lib_repo.path_to_oval_id(filepath) for filepath in changes}
# find all upstream ids
if verbose:
print(" ---- Locating parent definitions for all changed elements...")
elements_index = lib_search.ElementsIndex(False)
upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set())
# filter affected to definition ids
affected_def_ids = {
oval_id for oval_id in upstream_ids if lib_repo.get_element_type_from_oval_id(oval_id) == "definition"
}
# get all downstream elements
if verbose:
print(" ---- Resolving all elements needed to build comprehensive document...")
oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
if verbose:
print(" ---- Importing separate elements into comprehensive document....")
oval = OvalDocument(None)
for path in file_paths:
element = OvalElement.fromStandaloneFile(path)
if element is None:
print(":::: None from path: ", path)
return None
oval.addElement(element, True)
return etree.fromstring(oval.to_string())
def document_iterator(self, paths=False):
""" Iterator yielding definition revisions in repo as indexable documents. """
if not paths:
# get all in repo
paths = lib_repo.get_definition_paths_iterator()
for path in paths:
try:
document = lib_xml.get_definition_metadata(path)
except lib_xml.InvalidPathError as e:
yield { 'oval_id': lib_repo.path_to_oval_id(e.path), 'deleted': True }
except lib_xml.InvalidXmlError as e:
raise
else:
oval_id = document['oval_id']
for revision in document['revisions']:
revision['oval_id'] = oval_id
revision = self.whoosh_escape_document(revision)
yield revision
def show_affected(file_list):
"""For a list of files, show all elements that reference them
"""
if file_list is None or len(file_list) < 1:
print(" ----- Empty file list. Nothing is affected by that")
return
elements_index = lib_search.ElementsIndex(False)
for file in file_list:
ovalid = lib_repo.path_to_oval_id(file)
print("\n=========== For item {0}:".format(ovalid))
affected = elements_index.find_upstream_ids(ovalid, set(), depth_limit = 1)
if affected is not None and len(affected) > 0:
for affected_id in affected:
print(" ---> {0}".format(affected_id))
else:
print(" **** No affected items found ****")
def show_affected(file_list):
"""For a list of files, show all elements that reference them
"""
if file_list is None or len(file_list) < 1:
print(" ----- Empty file list. Nothing is affected by that")
return
elements_index = lib_search.ElementsIndex(False)
for file in file_list:
ovalid = lib_repo.path_to_oval_id(file)
print("\n=========== For item {0}:".format(ovalid))
affected = elements_index.find_upstream_ids(ovalid, set(), depth_limit = 1)
if affected is not None and len(affected) > 0:
for affected_id in affected:
print(" ---> {0}".format(affected_id))
else:
print(" **** No affected items found ****")
def document_iterator(self, paths=False):
""" Iterator yielding definition revisions in repo as indexable documents. """
if not paths:
# get all in repo
paths = lib_repo.get_definition_paths_iterator()
for path in paths:
try:
document = lib_xml.get_definition_metadata(path)
except lib_xml.InvalidPathError as e:
yield { 'oval_id': lib_repo.path_to_oval_id(e.path), 'deleted': True }
except lib_xml.InvalidXmlError as e:
raise
else:
oval_id = document['oval_id']
for revision in document['revisions']:
revision['oval_id'] = oval_id
revision = self.whoosh_escape_document(revision)
yield revision
def main():
orphans = {}
elements_index = lib_search.ElementsIndex(message)
for elempath in lib_repo.get_element_paths_iterator():
oval_id = lib_repo.path_to_oval_id(elempath)
oval_ids = { oval_id }
upstream_ids = elements_index.find_upstream_ids(oval_ids, set())
downstream_ids = elements_index.find_downstream_ids(oval_ids, set())
message("INFO", "OVAL ID '%s' had %i upstream ids and %i downstream ids" % (oval_id, len(upstream_ids), len(downstream_ids)))
if len(upstream_ids) == 0 and len(downstream_ids) == 0:
orphans[oval_id] = elempath
for ok in orphans.keys():
message("INFO", "Found Orphan '%s'" % ok)
os.remove(orphans[ok])
def main():
start_time = time.time()
definitions_index = lib_search.DefinitionsIndex(message)
ns_map = {'oval-def': 'http://oval.mitre.org/XMLSchema/oval-definitions-5'}
# Create the output object...
output = WebOutput()
for defpath in lib_repo.get_definition_paths_iterator():
def_id = lib_repo.path_to_oval_id(defpath)
try:
tree = etree.parse(defpath)
except OSError as e:
raise InvalidPathError(defpath)
except etree.XMLSyntaxError as e:
raise InvalidXmlError(defpath, e.msg)
root = tree.getroot()
# Only Vulnerability and Patch definitions would have CVE references...
defclass = root.get('class')
if defclass in ["vulnerability", "patch"]:
reference = root.find(
"./oval-def:metadata/oval-def:reference[@source='CVE']",
ns_map)
if reference is not None:
output.add_result(
def_id,
[reference.get("ref_id"),
reference.get("ref_url")])
# add timing
seconds_elapsed = time.time() - start_time
output.message("CVE", "OVAL Repository Definition-to-CVE Mappings")
output.message("time", "{0}".format(format_duration(seconds_elapsed)))
# Write JSON Output...
output.write_json()
def main():
orphans = {}
elements_index = lib_search.ElementsIndex(message)
for elempath in lib_repo.get_element_paths_iterator():
oval_id = lib_repo.path_to_oval_id(elempath)
oval_ids = {oval_id}
upstream_ids = elements_index.find_upstream_ids(oval_ids, set())
downstream_ids = elements_index.find_downstream_ids(oval_ids, set())
message(
"INFO", "OVAL ID '%s' had %i upstream ids and %i downstream ids" %
(oval_id, len(upstream_ids), len(downstream_ids)))
if len(upstream_ids) == 0 and len(downstream_ids) == 0:
orphans[oval_id] = elempath
for ok in orphans.keys():
message("INFO", "Found Orphan '%s'" % ok)
os.remove(orphans[ok])
def main():
""" parse command line options and call lib functions """
start_time = time.time()
parser = argparse.ArgumentParser(description='Identify changes in current working directory as compared to a remote authoritative copy of the repo and identify all the elements affected by those changes.')
# Note: I don't think we need to support files. If a file is submitted, CIS/QA can decompose it and then run this. So this can always run against repo.
#parser.add_argument('-f', '--file', required=False, help='The name of the source file. If not used the local git repository will be used as the source')
parser.add_argument('--silent', required=False, action="store_true", help='Suppress messages')
parser.add_argument('--remote', required=False, default='upstream', help="name of authoritative remote (default: 'upstream')")
parser.add_argument('--branch', required=False, default='master', help="name of branch in authoritative remote (default: 'master')")
parser.add_argument('--outfile', required=False, default='all.affected.oval.xml', help="file name OVAL definitions file containing all affected definitions (default 'all.affected.oval.xml')")
parser.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the affected definitions')
args = vars(parser.parse_args())
silent = args['silent']
## 1. Find Affected Elements
# get changes in working dir vs. remote/branch
message('info', 'Comparing working directory to {0}/{1}'.format(args['remote'], args['branch']), silent)
paths_changed = lib_git.compare_current_oval_to_remote(args['remote'], args['branch'])
if not paths_changed:
message('info', 'No changes. Aborting.', silent)
sys.exit(0)
message('info', 'Found {0} files changed in working directory:\n\t{1}'.format(len(paths_changed), '\n\t'.join(paths_changed)), silent)
# convert paths to oval ids
oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in paths_changed }
message('info', 'Found {0} OVAL elements changed in working directory:\n\t{1}'.format(len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent)
# find all upstream ids
message('info','Finding upstream OVAL ids for {0} element(s)'.format(len(oval_ids_changed)), silent)
elements_index = lib_search.ElementsIndex(message)
upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set())
message('info','Found {0} upstream OVAL ids (all element types)'.format(len(upstream_ids)), silent)
affected_oval_ids = oval_ids_changed.union(upstream_ids)
def main():
""" parse command line options and call lib functions """
start_time = time.time()
parser = argparse.ArgumentParser(description='Identify changes in current working directory as compared to a remote authoritative copy of the repo and identify all the elements affected by those changes.')
# Note: I don't think we need to support files. If a file is submitted, CIS/QA can decompose it and then run this. So this can always run against repo.
#parser.add_argument('-f', '--file', required=False, help='The name of the source file. If not used the local git repository will be used as the source')
parser.add_argument('--silent', required=False, action="store_true", help='Suppress messages')
parser.add_argument('--remote', required=False, default='upstream', help="name of authoritative remote (default: 'upstream')")
parser.add_argument('--branch', required=False, default='master', help="name of branch in authoritative remote (default: 'master')")
parser.add_argument('--outfile', required=False, default='all.affected.oval.xml', help="file name OVAL definitions file containing all affected definitions (default 'all.affected.oval.xml')")
parser.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the affected definitions')
args = vars(parser.parse_args())
silent = args['silent']
## 1. Find Affected Elements
# get changes in working dir vs. remote/branch
message('info', 'Comparing working directory to {0}/{1}'.format(args['remote'], args['branch']), silent)
paths_changed = lib_git.compare_current_oval_to_remote(args['remote'], args['branch'])
if not paths_changed:
message('info', 'No changes. Aborting.', silent)
sys.exit(0)
message('info', 'Found {0} files changed in working directory:\n\t{1}'.format(len(paths_changed), '\n\t'.join(paths_changed)), silent)
# convert paths to oval ids
oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in paths_changed }
message('info', 'Found {0} OVAL elements changed in working directory:\n\t{1}'.format(len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent)
# find all upstream ids
message('info','Finding upstream OVAL ids for {0} element(s)'.format(len(oval_ids_changed)), silent)
elements_index = lib_search.ElementsIndex(message)
upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set())
message('info','Found {0} upstream OVAL ids (all element types)'.format(len(upstream_ids)), silent)
affected_oval_ids = oval_ids_changed.union(upstream_ids)
# filter affected to defintion ids
affected_def_ids = { oval_id for oval_id in affected_oval_ids if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition' }
message('info','Found {0} upstream OVAL definitions:\n\t{1}'.format(len(affected_def_ids), '\n\t'.join(affected_def_ids)), silent)
## 2. Build an OVAL Definitions File and Validate It!
message('info','Building an OVAL definitions file for all affected definitions.', silent)
# get all downstream elements
oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
# add each OVAL definition to generator and write to file
message('info',"Generating OVAL definition file '{0}' with {1} elements".format(args['outfile'], len(oval_ids)), silent)
OvalGenerator = lib_xml.OvalGenerator(message)
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
OvalGenerator.to_file(args['outfile'])
# validate
schema_path = lib_repo.get_oval_def_schema('5.11.1')
message('info','Performing schema validation', silent)
try:
lib_xml.schema_validate(args['outfile'], schema_path)
message('info','Schema validation successful', silent)
except lib_xml.SchemaValidationError as e:
message('error','Schema validation failed:\n\t{0}'.format(e.message), silent)
if args['schematron']:
# schematron validate
schema_path = lib_repo.get_oval_def_schema('5.11.1')
message('info','Performing schematron validation', silent)
try:
lib_xml.schematron_validate(args['outfile'], schema_path)
message('info','Schematron validation successful', silent)
except lib_xml.SchematronValidationError as e:
message('error','Schematron validation failed:\n\t{0}'.format('\n\t'.join(e.messages)), silent)
#Find all downstream children -- that is, a search depth of one
#Find all upstream users, all the way up to the definition
#Sort the list: definitions, then tests, objects, states, and variables
#Show the list
#Offer to build an OVAL file that contains all the changes
seconds_elapsed = time.time() - start_time
message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)), silent)
def main():
start_time = time.time()
tracking = dict()
elements_index = lib_search.ElementsIndex(message)
# getting all definitions
all_def_ids = set()
message('info', 'getting all definitions')
for defpath in lib_repo.get_definition_paths_iterator():
all_def_ids.add(lib_repo.path_to_oval_id(defpath))
message('info', 'found {0} definitions'.format(len(all_def_ids)))
# getting all element ids downstream from any definition
message('info', 'getting all downstream element ids')
all_downstream_ids = elements_index.find_downstream_ids(all_def_ids)
message('info',
'found {0} downstream element ids'.format(len(all_downstream_ids)))
# get elements that aren't in all_downstream_ids
message('info', 'checking all elements')
cur_element_type = None
for elempath in lib_repo.get_element_paths_iterator():
oval_id = lib_repo.path_to_oval_id(elempath)
element_type = lib_repo.get_element_type_from_oval_id(oval_id)
# skip definitions... we're only pruning child elements
if element_type == 'definition':
continue
# write status msg
if element_type != cur_element_type:
cur_element_type = element_type
i_element_type = 0
i_element_type = i_element_type + 1
sys.stdout.write(
'Analyzing {0}s: {1} \r'.format(
cur_element_type, i_element_type))
sys.stdout.flush()
# it's an orphan if it's not downsteam of a definition
track_as = 'orphan' if oval_id not in all_downstream_ids else 'in_use'
if not track_as in tracking:
tracking[track_as] = dict()
if not element_type in tracking[track_as]:
tracking[track_as][element_type] = set()
tracking[track_as][element_type].add(oval_id)
sys.stdout.write(
' \r'.
format(cur_element_type, i_element_type))
sys.stdout.flush()
# generate report
report = []
for track_as, elements_by_type in tracking.items():
report.append('\t{0}:'.format(track_as.replace('_', ' ').capitalize()))
for element_type, oval_ids in elements_by_type.items():
report.append('\t\t{0} {1}s'.format(len(oval_ids), element_type))
message('found', '\n'.join(report))
response = input("\n :::: Remove all orphans? (N[o] / y[es]): ")
if response.lower() == 'y':
orphan_ids = set()
for element_type, oval_ids in tracking['orphan'].items():
orphan_ids.update(oval_ids)
file_paths = elements_index.get_paths_from_ids(orphan_ids)
for file_path in file_paths:
message("INFO",
"Deleting Orphan '%s'" % os.path.basename(file_path))
os.remove(file_path)
seconds_elapsed = time.time() - start_time
message('info',
'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def create_definition(self, cve_id, summary, test_ref_path, new_def_path):
self.cve_id = cve_id
self.summary = summary
self.test_ref_id = lib_repo.path_to_oval_id(test_ref_path)
self.new_def_path = new_def_path
self.def_id = lib_repo.path_to_oval_id(new_def_path)
self.def_platform = 'GenericPlatform'
self.criteria_comment = 'Automatically added for NVD CVE {0}'.format(
self.cve_id)
self.def_family = 'Linux'
self.def_cve_ref_url = mitre_cve_url + self.cve_id
self.def_criterion_comment = 'Automatically generated for NVD CVE {0}'.format(
self.cve_id)
self.date = date.today().strftime('%Y-%m-%dT%H:%M:%S%z')
doc_root = etree.fromstring(oval_definition)
for node in doc_root.getiterator():
if self.namespace + 'definition' in node.tag:
node.set('id', self.def_id)
if self.namespace + 'title' in node.tag:
node.text = self.summary
if self.namespace + 'affected' in node.tag:
node.set('family', self.def_family)
if self.namespace + 'platform' in node.tag:
node.text = self.def_platform
if self.namespace + 'reference' in node.tag:
node.set('ref_id', self.cve_id)
node.set('ref_url', self.def_cve_ref_url)
if self.namespace + 'description' in node.tag:
node.text = self.summary
if self.namespace + 'submitted' in node.tag:
node.set('date', str(self.date))
if self.namespace + 'contributor' in node.tag:
node.set('organization', 'Automatically generated from NVD')
node.text = 'Automatically generated from NVD'
if self.namespace + 'status_change' in node.tag:
node.set('date', str(self.date))
node.text = 'ACCEPTED'
if self.namespace + 'status' in node.tag:
node.text = 'ACCEPTED'
if self.namespace + 'criterion' in node.tag:
node.set(
'comment',
'Automatically added for NVD CVE {0}'.format(self.cve_id))
node.set('test_ref', self.test_ref_id)
NVDOVALXMLWriter().write(self.new_def_path, doc_root)
return
def main():
""" parse command line options and call lib functions """
start_time = time.time()
parser = argparse.ArgumentParser(
description=
'Identify changes in current working directory as compared to a remote authoritative copy of the repo and identify all the elements affected by those changes.'
)
# Note: I don't think we need to support files. If a file is submitted, CIS/QA can decompose it and then run this. So this can always run against repo.
#parser.add_argument('-f', '--file', required=False, help='The name of the source file. If not used the local git repository will be used as the source')
parser.add_argument('--silent',
required=False,
action="store_true",
help='Suppress messages')
parser.add_argument(
'--remote',
required=False,
default='upstream',
help="name of authoritative remote (default: 'upstream')")
parser.add_argument(
'--branch',
required=False,
default='master',
help="name of branch in authoritative remote (default: 'master')")
parser.add_argument(
'--outfile',
required=False,
default='all.affected.oval.xml',
help=
"file name OVAL definitions file containing all affected definitions (default 'all.affected.oval.xml')"
)
parser.add_argument('-s',
'--schematron',
default=False,
action="store_true",
help='schematron validate the affected definitions')
args = vars(parser.parse_args())
silent = args['silent']
## 1. Find Affected Elements
# get changes in working dir vs. remote/branch
message(
'info', 'Comparing working directory to {0}/{1}'.format(
args['remote'], args['branch']), silent)
paths_changed = lib_git.compare_current_oval_to_remote(
args['remote'], args['branch'])
if not paths_changed:
message('info', 'No changes. Aborting.', silent)
sys.exit(0)
message(
'info', 'Found {0} files changed in working directory:\n\t{1}'.format(
len(paths_changed), '\n\t'.join(paths_changed)), silent)
# convert paths to oval ids
oval_ids_changed = {
lib_repo.path_to_oval_id(filepath)
for filepath in paths_changed
}
message(
'info',
'Found {0} OVAL elements changed in working directory:\n\t{1}'.format(
len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent)
# find all upstream ids
message(
'info', 'Finding upstream OVAL ids for {0} element(s)'.format(
len(oval_ids_changed)), silent)
elements_index = lib_search.ElementsIndex(message)
upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set())
message(
'info', 'Found {0} upstream OVAL ids (all element types)'.format(
len(upstream_ids)), silent)
affected_oval_ids = oval_ids_changed.union(upstream_ids)
# filter affected to definition ids
affected_def_ids = {
oval_id
for oval_id in upstream_ids
if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition'
}
message(
'info', 'Found {0} upstream OVAL definitions:\n\t{1}'.format(
len(affected_def_ids), '\n\t'.join(affected_def_ids)), silent)
## 2. Build an OVAL Definitions File and Validate It!
message('info',
'Building an OVAL definitions file for all affected definitions.',
silent)
# get all downstream elements
oval_ids = elements_index.find_downstream_ids(affected_def_ids,
affected_def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
# add each OVAL definition to generator and write to file
message(
'info',
"Generating OVAL definition file '{0}' with {1} elements".format(
args['outfile'], len(oval_ids)), silent)
OvalGenerator = lib_xml.OvalGenerator(message)
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
OvalGenerator.to_file(args['outfile'])
# validate
schema_path = lib_repo.get_oval_def_schema('5.11.1')
message('info', 'Performing schema validation', silent)
try:
lib_xml.schema_validate(args['outfile'], schema_path)
message('info', 'Schema validation successful', silent)
except lib_xml.SchemaValidationError as e:
message('error', 'Schema validation failed:\n\t{0}'.format(e.message),
silent)
if args['schematron']:
# schematron validate
schema_path = lib_repo.get_oval_def_schema('5.11.1')
message('info', 'Performing schematron validation', silent)
try:
lib_xml.schematron_validate(args['outfile'], schema_path)
message('info', 'Schematron validation successful', silent)
except lib_xml.SchematronValidationError as e:
message(
'error', 'Schematron validation failed:\n\t{0}'.format(
'\n\t'.join(e.messages)), silent)
#Find all downstream children -- that is, a search depth of one
#Find all upstream users, all the way up to the definition
#Sort the list: definitions, then tests, objects, states, and variables
#Show the list
#Offer to build an OVAL file that contains all the changes
seconds_elapsed = time.time() - start_time
message('info',
'Completed in {0}!'.format(format_duration(seconds_elapsed)),
silent)
def main():
start_time = time.time()
tracking = dict()
elements_index = lib_search.ElementsIndex(message)
# getting all definitions
all_def_ids = set()
message('info', 'getting all definitions')
for defpath in lib_repo.get_definition_paths_iterator():
all_def_ids.add(lib_repo.path_to_oval_id(defpath))
message('info', 'found {0} definitions'.format(len(all_def_ids)))
# getting all element ids downstream from any definition
message('info', 'getting all downstream element ids')
all_downstream_ids = elements_index.find_downstream_ids(all_def_ids)
message('info', 'found {0} downstream element ids'.format(len(all_downstream_ids)))
# get elements that aren't in all_downstream_ids
message('info', 'checking all elements')
cur_element_type = None
for elempath in lib_repo.get_element_paths_iterator():
oval_id = lib_repo.path_to_oval_id(elempath)
element_type = lib_repo.get_element_type_from_oval_id(oval_id)
# skip definitions... we're only pruning child elements
if element_type == 'definition':
continue
# write status msg
if element_type != cur_element_type:
cur_element_type = element_type
i_element_type = 0
i_element_type = i_element_type + 1
sys.stdout.write('Analyzing {0}s: {1} \r'.format(cur_element_type, i_element_type))
sys.stdout.flush()
# it's an orphan if it's not downsteam of a definition
track_as = 'orphan' if oval_id not in all_downstream_ids else 'in_use'
if not track_as in tracking:
tracking[track_as] = dict()
if not element_type in tracking[track_as]:
tracking[track_as][element_type] = set()
tracking[track_as][element_type].add(oval_id)
sys.stdout.write(' \r'.format(cur_element_type, i_element_type))
sys.stdout.flush()
# generate report
report = []
for track_as, elements_by_type in tracking.items():
report.append('\t{0}:'.format(track_as.replace('_', ' ').capitalize()))
for element_type, oval_ids in elements_by_type.items():
report.append('\t\t{0} {1}s'.format(len(oval_ids), element_type))
message('found', '\n'.join(report))
response = input("\n :::: Remove all orphans? (N[o] / y[es]): ")
if response.lower() == 'y':
orphan_ids = set()
for element_type, oval_ids in tracking['orphan'].items():
orphan_ids.update(oval_ids)
file_paths = elements_index.get_paths_from_ids(orphan_ids)
for file_path in file_paths:
message("INFO", "Deleting Orphan '%s'" % os.path.basename(file_path))
os.remove(file_path)
seconds_elapsed = time.time() - start_time
message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
