def create_definition(self, cve_id, summary, test_ref_path, new_def_path): self.cve_id = cve_id self.summary = summary self.test_ref_id = lib_repo.path_to_oval_id(test_ref_path) self.new_def_path = new_def_path self.def_id = lib_repo.path_to_oval_id(new_def_path) self.def_platform = 'GenericPlatform' self.criteria_comment = 'Automatically added for NVD CVE {0}'.format(self.cve_id) self.def_family = 'Linux' self.def_cve_ref_url = mitre_cve_url + self.cve_id self.def_criterion_comment = 'Automatically generated for NVD CVE {0}'.format(self.cve_id) self.date = date.today().strftime('%Y-%m-%dT%H:%M:%S%z') doc_root = etree.fromstring(oval_definition) for node in doc_root.getiterator(): if self.namespace + 'definition' in node.tag: node.set('id', self.def_id) if self.namespace + 'title' in node.tag: node.text = self.summary if self.namespace + 'affected' in node.tag: node.set('family', self.def_family) if self.namespace + 'platform' in node.tag: node.text = self.def_platform if self.namespace + 'reference' in node.tag: node.set('ref_id', self.cve_id) node.set('ref_url', self.def_cve_ref_url) if self.namespace + 'description' in node.tag: node.text = self.summary if self.namespace + 'submitted' in node.tag: node.set('date', str(self.date)) if self.namespace + 'contributor' in node.tag: node.set('organization', 'Automatically generated from NVD') node.text = 'Automatically generated from NVD' if self.namespace + 'status_change' in node.tag: node.set('date', str(self.date)) node.text = 'ACCEPTED' if self.namespace + 'status' in node.tag: node.text = 'ACCEPTED' if self.namespace + 'criterion' in node.tag: node.set('comment', 'Automatically added for NVD CVE {0}'.format(self.cve_id)) node.set('test_ref', self.test_ref_id) NVDOVALXMLWriter().write(self.new_def_path, doc_root) return
def put(self, cve_id, test_path, object_path, state_path): self.test_path, self.object_path, self.state_path = test_path, object_path, state_path self.check = 'all' self.check_existence = 'at_least_one_exists' self.comment = 'Automatically added for NVD CVE {0}'.format(cve_id) self.test_id = lib_repo.path_to_oval_id(self.test_path) self.object_id = lib_repo.path_to_oval_id(self.object_path) self.state_id = lib_repo.path_to_oval_id(self.state_path) self.make() return
def main(): orphans = {} elements_index = lib_search.ElementsIndex(message) for elempath in lib_repo.get_element_paths_iterator(): oval_id = lib_repo.path_to_oval_id(elempath) oval_ids = { oval_id } upstream_ids = elements_index.find_upstream_ids(oval_ids, set()) downstream_ids = elements_index.find_downstream_ids(oval_ids, set()) #message("INFO", "OVAL ID '%s' had %i upstream ids and %i downstream ids" % (oval_id, len(upstream_ids), len(downstream_ids))) if len(upstream_ids) == 0 and len(downstream_ids) == 0: message("INFO", "Found Orphan '%s'" % oval_id) orphans[oval_id] = elempath response = input("\n :::: Remove all orphans? (N[o] / y[es]): ") if response != 'y' and response != 'Y': return message("INFO", "\n\n") for ok in orphans.keys(): message("INFO", "Deleting Orphan '%s'" % ok) os.remove(orphans[ok])
def main(): start_time = time.time() definitions_index = lib_search.DefinitionsIndex(message) ns_map = {"oval-def": "http://oval.mitre.org/XMLSchema/oval-definitions-5"} # Create the output object... output = WebOutput() for defpath in lib_repo.get_definition_paths_iterator(): def_id = lib_repo.path_to_oval_id(defpath) try: tree = etree.parse(defpath) except OSError as e: raise InvalidPathError(defpath) except etree.XMLSyntaxError as e: raise InvalidXmlError(defpath, e.msg) root = tree.getroot() # Only Vulnerability and Patch definitions would have CVE references... defclass = root.get("class") if defclass in ["vulnerability", "patch"]: reference = root.find("./oval-def:metadata/oval-def:reference[@source='CVE']", ns_map) if reference is not None: output.add_result(def_id, [reference.get("ref_id"), reference.get("ref_url")]) # add timing seconds_elapsed = time.time() - start_time output.message("CVE", "OVAL Repository Definition-to-CVE Mappings") output.message("time", "{0}".format(format_duration(seconds_elapsed))) # Write JSON Output... output.write_json()
def determine_def_min_version(defpath, definitions_index, elements_index, update): """ determines the minimum oval schema version the given definition can be expressed in. """ global schema_path_cache # build schema path cache for schema_version in lib_repo.get_schema_versions(): schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version) minimum_version=None exception=None # get id of oval definition def_id = lib_repo.path_to_oval_id(defpath) # add all downstream element ids def_ids = set([def_id]) oval_ids = elements_index.find_downstream_ids(def_ids, def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # create generator that builds files in memory (these are small) OvalGenerator = lib_xml.OvalGenerator(message) OvalGenerator.use_file_queues = False # add each OVAL definition to generator for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) # parse defintion, get ref to schema_version element xmlstring = OvalGenerator.to_string(True, False) xmlstring = xmlstring.replace('<?xml version="1.0" encoding="UTF-8"?>','') xmlstring = xmlstring.replace('<?xml version="1.0" encoding="utf-8"?>','') xmlstring = xmlstring.replace("<?xml version='1.0' encoding='UTF-8'?>",'') xmlstring = xmlstring.replace("<?xml version='1.0' encoding='utf-8'?>",'') #message("INFO", "XMLSTRING is %s" % xmlstring) #tree = etree.fromstring(OvalGenerator.to_string(True, False)) tree = etree.XML(xmlstring) schema_version_element = tree.find('.//oval:schema_version', { 'oval': 'http://oval.mitre.org/XMLSchema/oval-common-5' }) for schema_version in lib_repo.get_schema_versions(): # update schema version in tree schema_version_element.text = schema_version # test of definitions file validates against current schema try: message("INFO", "Schema Path is %s" % schema_path_cache[schema_version]) lib_xml.schema_validate(tree, schema_path_cache[schema_version]) minimum_version = schema_version message("INFO", "XML is valid against schema version %s" % schema_version) except lib_xml.SchemaValidationError as e: exception = e.message message("ERROR", "Exception (schema version is %s) - %s" % (schema_version, e.message)) break; if update: set_min_schema_version(defpath, minimum_version) return { 'oval_id': def_id, 'minimum_version': minimum_version, 'exception': exception }
def put(self, cve_id, xml_path, product): self.cve_id, self.xml_path, self.product = cve_id, xml_path, product self.comment = 'Automatically added for NVD CVE {0}'.format(cve_id) self.this_id = lib_repo.path_to_oval_id(self.xml_path) self.make() return
def get_last_def(self): for xml_file in lib_repo.get_element_paths_iterator(): if 'vulnerability' in xml_file: current_id = int(lib_repo.path_to_oval_id(xml_file).split(':')[3]) if current_id > self.new_id: self.new_id = current_id self.path = os.path.split(xml_file)[0] self.new_filename = os.path.split(xml_file)[1] return
def get_last(self): for xml_file in lib_repo.get_element_paths_iterator(): if (self.tst_or_obj_or_ste in xml_file) and (self.family in xml_file): current_id = int(lib_repo.path_to_oval_id(xml_file).split(':')[3]) if current_id > self.new_id: self.new_id = current_id self.path = os.path.split(xml_file)[0] self.new_filename = os.path.split(xml_file)[1] return
def get_last_def(self): for xml_file in lib_repo.get_element_paths_iterator(): if 'vulnerability' in xml_file: current_id = int( lib_repo.path_to_oval_id(xml_file).split(':')[3]) if current_id > self.new_id: self.new_id = current_id self.path = os.path.split(xml_file)[0] self.new_filename = os.path.split(xml_file)[1] return
def put(self, cve_id, summary, test_ref_path, new_def_path): self.cve_id = cve_id self.summary = summary self.test_ref_id = lib_repo.path_to_oval_id(test_ref_path) self.definition_id = lib_repo.path_to_oval_id(new_def_path) self.platform = 'GenericPlatform' self.criteria_comment = 'Automatically added for NVD CVE {0}'.format(self.cve_id) # check it the OVAL repository contains the CVE # if yes, update the definition with stub containing the definition created above if self.get_definition_path(): self.update_definition() else: NVDOVALDefinitionVulnerabilityCreate().create_definition(cve_id, summary, test_ref_path, new_def_path) # create new definition # NVDOVALDefinitionVulnerabilityCreate().create_definition(cve_id, summary, test_ref_path, new_def_path) return
def get_last(self): for xml_file in lib_repo.get_element_paths_iterator(): if (self.tst_or_obj_or_ste in xml_file) and (self.family in xml_file): current_id = int( lib_repo.path_to_oval_id(xml_file).split(':')[3]) if current_id > self.new_id: self.new_id = current_id self.path = os.path.split(xml_file)[0] self.new_filename = os.path.split(xml_file)[1] return
def build_comprehensive_oval_document(changes): """ Builds an XML tree which contains all elements affected by the changes """ global debug global verbose if changes is None or len(changes) < 1: return None if verbose: print(" ---- Getting OVAL ID's for all changed files...") oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in changes } # find all upstream ids if verbose: print( " ---- Locating parent definitions for all changed elements...") elements_index = lib_search.ElementsIndex(False) upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set()) # filter affected to definition ids affected_def_ids = { oval_id for oval_id in upstream_ids if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition' } # get all downstream elements if verbose: print( " ---- Resolving all elements needed to build comprehensive document..." ) oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) if verbose: print( " ---- Importing separate elements into comprehensive document...." ) oval = OvalDocument(None) for path in file_paths: element = OvalElement.fromStandaloneFile(path) if element is None: print(":::: None from path: ", path) return None oval.addElement(element, True) return etree.fromstring(oval.to_string())
def put(self, cve_id, xml_path, version): self.cve_id, self.xml_path, self.version = cve_id, xml_path, version self.comment = 'Automatically added for NVD CVE {0}'.format(cve_id) self.this_id = lib_repo.path_to_oval_id(self.xml_path) self.version = '0:' + self.version self.datatype = 'evr_string' self.operation = 'less than or equal' self.make() return
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser( description='Builds all OVAL definitons in the repository.') parser.add_argument( '-l', '--limit', nargs='?', default="0", type=int, help='limits number of definitions that will be built)') args = vars(parser.parse_args()) # get indexes # definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) # create generator, build in memory b/c smallish files OvalGenerator = lib_xml.OvalGenerator(message) OvalGenerator.use_file_queues = False print("Storing data in temp directory: {}".format(tempfile.gettempdir())) i_file = 0 i_limit = args['limit'] for defpath in lib_repo.get_definition_paths_iterator(): i_file = i_file + 1 if i_limit > 0 and i_file > i_limit: break def_id = lib_repo.path_to_oval_id(defpath) message('info', 'Building file {0} for {1}.'.format(i_file, def_id)) # add all downstream element ids def_ids = set([def_id]) oval_ids = elements_index.find_downstream_ids(def_ids, def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # add each OVAL definition to generator for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) # write output file outfile = '{1}/gen.{0}'.format(lib_repo.oval_id_to_path(def_id), tempfile.gettempdir()) OvalGenerator.to_file(outfile) seconds_elapsed = time.time() - start_time message('info', 'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def determine_def_min_version(defpath, definitions_index, elements_index, update): """ determines the minimum oval schema version the given definition can be expressed in. """ global schema_path_cache # build schema path cache for schema_version in lib_repo.get_schema_versions(): schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version) minimum_version=None exception=None # get id of oval definition def_id = lib_repo.path_to_oval_id(defpath) # add all downstream element ids def_ids = set([def_id]) oval_ids = elements_index.find_downstream_ids(def_ids, def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # create generator that builds files in memory (these are small) OvalGenerator = lib_xml.OvalGenerator(message) OvalGenerator.use_file_queues = False # add each OVAL definition to generator for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) # parse defintion, get ref to schema_version element tree = etree.fromstring(OvalGenerator.to_string(True, False)) schema_version_element = tree.find('.//oval:schema_version', { 'oval': 'http://oval.mitre.org/XMLSchema/oval-common-5' }) for schema_version in lib_repo.get_schema_versions(): # update schema version in tree schema_version_element.text = schema_version # test of definitions file validates against current schema try: message("INFO", "Schema Path is %s" % schema_path_cache[schema_version]) lib_xml.schema_validate(tree, schema_path_cache[schema_version]) minimum_version = schema_version message("INFO", "XML is valid against schema version %s" % schema_version) except lib_xml.SchemaValidationError as e: exception = e.message message("ERROR", "Exception (schema version is %s) - %s" % (schema_version, e.message)) break; if update: set_min_schema_version(defpath, minimum_version) return { 'oval_id': def_id, 'minimum_version': minimum_version, 'exception': exception }
def put(self, cve_id, summary, test_ref_path, new_def_path): self.cve_id = cve_id self.summary = summary self.test_ref_id = lib_repo.path_to_oval_id(test_ref_path) self.definition_id = lib_repo.path_to_oval_id(new_def_path) self.platform = 'GenericPlatform' self.criteria_comment = 'Automatically added for NVD CVE {0}'.format( self.cve_id) # check it the OVAL repository contains the CVE # if yes, update the definition with stub containing the definition created above if self.get_definition_path(): self.update_definition() else: NVDOVALDefinitionVulnerabilityCreate().create_definition( cve_id, summary, test_ref_path, new_def_path) # create new definition # NVDOVALDefinitionVulnerabilityCreate().create_definition(cve_id, summary, test_ref_path, new_def_path) return
def get_changed_ids_from_git(): try: changed_files = lib_git.get_uncommitted_oval() if not changed_files or changed_files is None: return None change_list = [] for file in changed_files: change_list.add(lib_repo.path_to_oval_id(file)) return change_list except Exception: if main.verbose: print("## Error while querying git for changes: ", format(Exception)) return None
def determine_def_min_version(defpath, definitions_index, elements_index, update): """ determines the minimum oval schema version the given definition can be expressed in. """ global schema_path_cache minimum_version = None exception = None # get id of oval definition def_id = lib_repo.path_to_oval_id(defpath) # add all downstream element ids def_ids = set([def_id]) oval_ids = elements_index.find_downstream_ids(def_ids, def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # create generator that builds files in memory (these are small) OvalGenerator = lib_xml.OvalGenerator(message) OvalGenerator.use_file_queues = False # add each OVAL definition to generator for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) # parse defintion, get ref to schema_version element tree = etree.fromstring(OvalGenerator.to_string()) schema_version_element = tree.find( ".//oval:schema_version", {"oval": "http://oval.mitre.org/XMLSchema/oval-common-5"} ) for schema_version in lib_repo.get_schema_versions(): # update schema version in tree schema_version_element.text = schema_version # test of definitions file validates against current schema try: lib_xml.schema_validate(tree, schema_path_cache[schema_version]) minimum_version = schema_version except lib_xml.SchemaValidationError as e: exception = e.message break if update: set_min_schema_version(defpath, minimum_version) return {"oval_id": def_id, "minimum_version": minimum_version, "exception": exception}
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser(description="Builds all OVAL definitons in the repository.") parser.add_argument( "-l", "--limit", nargs="?", default="0", type=int, help="limits number of definitions that will be built)" ) args = vars(parser.parse_args()) # get indexes definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) # create generator, build in memory b/c smallish files OvalGenerator = lib_xml.OvalGenerator(message) OvalGenerator.use_file_queues = False i_file = 0 i_limit = args["limit"] for defpath in lib_repo.get_definition_paths_iterator(): i_file = i_file + 1 if i_limit > 0 and i_file > i_limit: break def_id = lib_repo.path_to_oval_id(defpath) message("info", "Building file {0} for {1}.".format(i_file, def_id)) # add all downstream element ids def_ids = set([def_id]) oval_ids = elements_index.find_downstream_ids(def_ids, def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # add each OVAL definition to generator for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) # write output file outfile = "./tmp/gen.{0}".format(lib_repo.oval_id_to_path(def_id)) OvalGenerator.to_file(outfile) seconds_elapsed = time.time() - start_time message("info", "Completed in {0}!".format(format_duration(seconds_elapsed)))
def build_comprehensive_oval_document(changes): """ Builds an XML tree which contains all elements affected by the changes """ global debug global verbose if changes is None or len(changes) < 1: return None if verbose: print(" ---- Getting OVAL ID's for all changed files...") oval_ids_changed = {lib_repo.path_to_oval_id(filepath) for filepath in changes} # find all upstream ids if verbose: print(" ---- Locating parent definitions for all changed elements...") elements_index = lib_search.ElementsIndex(False) upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set()) # filter affected to definition ids affected_def_ids = { oval_id for oval_id in upstream_ids if lib_repo.get_element_type_from_oval_id(oval_id) == "definition" } # get all downstream elements if verbose: print(" ---- Resolving all elements needed to build comprehensive document...") oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) if verbose: print(" ---- Importing separate elements into comprehensive document....") oval = OvalDocument(None) for path in file_paths: element = OvalElement.fromStandaloneFile(path) if element is None: print(":::: None from path: ", path) return None oval.addElement(element, True) return etree.fromstring(oval.to_string())
def document_iterator(self, paths=False): """ Iterator yielding definition revisions in repo as indexable documents. """ if not paths: # get all in repo paths = lib_repo.get_definition_paths_iterator() for path in paths: try: document = lib_xml.get_definition_metadata(path) except lib_xml.InvalidPathError as e: yield { 'oval_id': lib_repo.path_to_oval_id(e.path), 'deleted': True } except lib_xml.InvalidXmlError as e: raise else: oval_id = document['oval_id'] for revision in document['revisions']: revision['oval_id'] = oval_id revision = self.whoosh_escape_document(revision) yield revision
def show_affected(file_list): """For a list of files, show all elements that reference them """ if file_list is None or len(file_list) < 1: print(" ----- Empty file list. Nothing is affected by that") return elements_index = lib_search.ElementsIndex(False) for file in file_list: ovalid = lib_repo.path_to_oval_id(file) print("\n=========== For item {0}:".format(ovalid)) affected = elements_index.find_upstream_ids(ovalid, set(), depth_limit = 1) if affected is not None and len(affected) > 0: for affected_id in affected: print(" ---> {0}".format(affected_id)) else: print(" **** No affected items found ****")
def main(): orphans = {} elements_index = lib_search.ElementsIndex(message) for elempath in lib_repo.get_element_paths_iterator(): oval_id = lib_repo.path_to_oval_id(elempath) oval_ids = { oval_id } upstream_ids = elements_index.find_upstream_ids(oval_ids, set()) downstream_ids = elements_index.find_downstream_ids(oval_ids, set()) message("INFO", "OVAL ID '%s' had %i upstream ids and %i downstream ids" % (oval_id, len(upstream_ids), len(downstream_ids))) if len(upstream_ids) == 0 and len(downstream_ids) == 0: orphans[oval_id] = elempath for ok in orphans.keys(): message("INFO", "Found Orphan '%s'" % ok) os.remove(orphans[ok])
def main(): start_time = time.time() definitions_index = lib_search.DefinitionsIndex(message) ns_map = {'oval-def': 'http://oval.mitre.org/XMLSchema/oval-definitions-5'} # Create the output object... output = WebOutput() for defpath in lib_repo.get_definition_paths_iterator(): def_id = lib_repo.path_to_oval_id(defpath) try: tree = etree.parse(defpath) except OSError as e: raise InvalidPathError(defpath) except etree.XMLSyntaxError as e: raise InvalidXmlError(defpath, e.msg) root = tree.getroot() # Only Vulnerability and Patch definitions would have CVE references... defclass = root.get('class') if defclass in ["vulnerability", "patch"]: reference = root.find( "./oval-def:metadata/oval-def:reference[@source='CVE']", ns_map) if reference is not None: output.add_result( def_id, [reference.get("ref_id"), reference.get("ref_url")]) # add timing seconds_elapsed = time.time() - start_time output.message("CVE", "OVAL Repository Definition-to-CVE Mappings") output.message("time", "{0}".format(format_duration(seconds_elapsed))) # Write JSON Output... output.write_json()
def main(): orphans = {} elements_index = lib_search.ElementsIndex(message) for elempath in lib_repo.get_element_paths_iterator(): oval_id = lib_repo.path_to_oval_id(elempath) oval_ids = {oval_id} upstream_ids = elements_index.find_upstream_ids(oval_ids, set()) downstream_ids = elements_index.find_downstream_ids(oval_ids, set()) message( "INFO", "OVAL ID '%s' had %i upstream ids and %i downstream ids" % (oval_id, len(upstream_ids), len(downstream_ids))) if len(upstream_ids) == 0 and len(downstream_ids) == 0: orphans[oval_id] = elempath for ok in orphans.keys(): message("INFO", "Found Orphan '%s'" % ok) os.remove(orphans[ok])
def main(): """ parse command line options and call lib functions """ start_time = time.time() parser = argparse.ArgumentParser(description='Identify changes in current working directory as compared to a remote authoritative copy of the repo and identify all the elements affected by those changes.') # Note: I don't think we need to support files. If a file is submitted, CIS/QA can decompose it and then run this. So this can always run against repo. #parser.add_argument('-f', '--file', required=False, help='The name of the source file. If not used the local git repository will be used as the source') parser.add_argument('--silent', required=False, action="store_true", help='Suppress messages') parser.add_argument('--remote', required=False, default='upstream', help="name of authoritative remote (default: 'upstream')") parser.add_argument('--branch', required=False, default='master', help="name of branch in authoritative remote (default: 'master')") parser.add_argument('--outfile', required=False, default='all.affected.oval.xml', help="file name OVAL definitions file containing all affected definitions (default 'all.affected.oval.xml')") parser.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the affected definitions') args = vars(parser.parse_args()) silent = args['silent'] ## 1. Find Affected Elements # get changes in working dir vs. remote/branch message('info', 'Comparing working directory to {0}/{1}'.format(args['remote'], args['branch']), silent) paths_changed = lib_git.compare_current_oval_to_remote(args['remote'], args['branch']) if not paths_changed: message('info', 'No changes. Aborting.', silent) sys.exit(0) message('info', 'Found {0} files changed in working directory:\n\t{1}'.format(len(paths_changed), '\n\t'.join(paths_changed)), silent) # convert paths to oval ids oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in paths_changed } message('info', 'Found {0} OVAL elements changed in working directory:\n\t{1}'.format(len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent) # find all upstream ids message('info','Finding upstream OVAL ids for {0} element(s)'.format(len(oval_ids_changed)), silent) elements_index = lib_search.ElementsIndex(message) upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set()) message('info','Found {0} upstream OVAL ids (all element types)'.format(len(upstream_ids)), silent) affected_oval_ids = oval_ids_changed.union(upstream_ids)
def main(): """ parse command line options and call lib functions """ start_time = time.time() parser = argparse.ArgumentParser(description='Identify changes in current working directory as compared to a remote authoritative copy of the repo and identify all the elements affected by those changes.') # Note: I don't think we need to support files. If a file is submitted, CIS/QA can decompose it and then run this. So this can always run against repo. #parser.add_argument('-f', '--file', required=False, help='The name of the source file. If not used the local git repository will be used as the source') parser.add_argument('--silent', required=False, action="store_true", help='Suppress messages') parser.add_argument('--remote', required=False, default='upstream', help="name of authoritative remote (default: 'upstream')") parser.add_argument('--branch', required=False, default='master', help="name of branch in authoritative remote (default: 'master')") parser.add_argument('--outfile', required=False, default='all.affected.oval.xml', help="file name OVAL definitions file containing all affected definitions (default 'all.affected.oval.xml')") parser.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the affected definitions') args = vars(parser.parse_args()) silent = args['silent'] ## 1. Find Affected Elements # get changes in working dir vs. remote/branch message('info', 'Comparing working directory to {0}/{1}'.format(args['remote'], args['branch']), silent) paths_changed = lib_git.compare_current_oval_to_remote(args['remote'], args['branch']) if not paths_changed: message('info', 'No changes. Aborting.', silent) sys.exit(0) message('info', 'Found {0} files changed in working directory:\n\t{1}'.format(len(paths_changed), '\n\t'.join(paths_changed)), silent) # convert paths to oval ids oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in paths_changed } message('info', 'Found {0} OVAL elements changed in working directory:\n\t{1}'.format(len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent) # find all upstream ids message('info','Finding upstream OVAL ids for {0} element(s)'.format(len(oval_ids_changed)), silent) elements_index = lib_search.ElementsIndex(message) upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set()) message('info','Found {0} upstream OVAL ids (all element types)'.format(len(upstream_ids)), silent) affected_oval_ids = oval_ids_changed.union(upstream_ids) # filter affected to defintion ids affected_def_ids = { oval_id for oval_id in affected_oval_ids if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition' } message('info','Found {0} upstream OVAL definitions:\n\t{1}'.format(len(affected_def_ids), '\n\t'.join(affected_def_ids)), silent) ## 2. Build an OVAL Definitions File and Validate It! message('info','Building an OVAL definitions file for all affected definitions.', silent) # get all downstream elements oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # add each OVAL definition to generator and write to file message('info',"Generating OVAL definition file '{0}' with {1} elements".format(args['outfile'], len(oval_ids)), silent) OvalGenerator = lib_xml.OvalGenerator(message) for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) OvalGenerator.to_file(args['outfile']) # validate schema_path = lib_repo.get_oval_def_schema('5.11.1') message('info','Performing schema validation', silent) try: lib_xml.schema_validate(args['outfile'], schema_path) message('info','Schema validation successful', silent) except lib_xml.SchemaValidationError as e: message('error','Schema validation failed:\n\t{0}'.format(e.message), silent) if args['schematron']: # schematron validate schema_path = lib_repo.get_oval_def_schema('5.11.1') message('info','Performing schematron validation', silent) try: lib_xml.schematron_validate(args['outfile'], schema_path) message('info','Schematron validation successful', silent) except lib_xml.SchematronValidationError as e: message('error','Schematron validation failed:\n\t{0}'.format('\n\t'.join(e.messages)), silent) #Find all downstream children -- that is, a search depth of one #Find all upstream users, all the way up to the definition #Sort the list: definitions, then tests, objects, states, and variables #Show the list #Offer to build an OVAL file that contains all the changes seconds_elapsed = time.time() - start_time message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)), silent)
def main(): start_time = time.time() tracking = dict() elements_index = lib_search.ElementsIndex(message) # getting all definitions all_def_ids = set() message('info', 'getting all definitions') for defpath in lib_repo.get_definition_paths_iterator(): all_def_ids.add(lib_repo.path_to_oval_id(defpath)) message('info', 'found {0} definitions'.format(len(all_def_ids))) # getting all element ids downstream from any definition message('info', 'getting all downstream element ids') all_downstream_ids = elements_index.find_downstream_ids(all_def_ids) message('info', 'found {0} downstream element ids'.format(len(all_downstream_ids))) # get elements that aren't in all_downstream_ids message('info', 'checking all elements') cur_element_type = None for elempath in lib_repo.get_element_paths_iterator(): oval_id = lib_repo.path_to_oval_id(elempath) element_type = lib_repo.get_element_type_from_oval_id(oval_id) # skip definitions... we're only pruning child elements if element_type == 'definition': continue # write status msg if element_type != cur_element_type: cur_element_type = element_type i_element_type = 0 i_element_type = i_element_type + 1 sys.stdout.write( 'Analyzing {0}s: {1} \r'.format( cur_element_type, i_element_type)) sys.stdout.flush() # it's an orphan if it's not downsteam of a definition track_as = 'orphan' if oval_id not in all_downstream_ids else 'in_use' if not track_as in tracking: tracking[track_as] = dict() if not element_type in tracking[track_as]: tracking[track_as][element_type] = set() tracking[track_as][element_type].add(oval_id) sys.stdout.write( ' \r'. format(cur_element_type, i_element_type)) sys.stdout.flush() # generate report report = [] for track_as, elements_by_type in tracking.items(): report.append('\t{0}:'.format(track_as.replace('_', ' ').capitalize())) for element_type, oval_ids in elements_by_type.items(): report.append('\t\t{0} {1}s'.format(len(oval_ids), element_type)) message('found', '\n'.join(report)) response = input("\n :::: Remove all orphans? (N[o] / y[es]): ") if response.lower() == 'y': orphan_ids = set() for element_type, oval_ids in tracking['orphan'].items(): orphan_ids.update(oval_ids) file_paths = elements_index.get_paths_from_ids(orphan_ids) for file_path in file_paths: message("INFO", "Deleting Orphan '%s'" % os.path.basename(file_path)) os.remove(file_path) seconds_elapsed = time.time() - start_time message('info', 'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def create_definition(self, cve_id, summary, test_ref_path, new_def_path): self.cve_id = cve_id self.summary = summary self.test_ref_id = lib_repo.path_to_oval_id(test_ref_path) self.new_def_path = new_def_path self.def_id = lib_repo.path_to_oval_id(new_def_path) self.def_platform = 'GenericPlatform' self.criteria_comment = 'Automatically added for NVD CVE {0}'.format( self.cve_id) self.def_family = 'Linux' self.def_cve_ref_url = mitre_cve_url + self.cve_id self.def_criterion_comment = 'Automatically generated for NVD CVE {0}'.format( self.cve_id) self.date = date.today().strftime('%Y-%m-%dT%H:%M:%S%z') doc_root = etree.fromstring(oval_definition) for node in doc_root.getiterator(): if self.namespace + 'definition' in node.tag: node.set('id', self.def_id) if self.namespace + 'title' in node.tag: node.text = self.summary if self.namespace + 'affected' in node.tag: node.set('family', self.def_family) if self.namespace + 'platform' in node.tag: node.text = self.def_platform if self.namespace + 'reference' in node.tag: node.set('ref_id', self.cve_id) node.set('ref_url', self.def_cve_ref_url) if self.namespace + 'description' in node.tag: node.text = self.summary if self.namespace + 'submitted' in node.tag: node.set('date', str(self.date)) if self.namespace + 'contributor' in node.tag: node.set('organization', 'Automatically generated from NVD') node.text = 'Automatically generated from NVD' if self.namespace + 'status_change' in node.tag: node.set('date', str(self.date)) node.text = 'ACCEPTED' if self.namespace + 'status' in node.tag: node.text = 'ACCEPTED' if self.namespace + 'criterion' in node.tag: node.set( 'comment', 'Automatically added for NVD CVE {0}'.format(self.cve_id)) node.set('test_ref', self.test_ref_id) NVDOVALXMLWriter().write(self.new_def_path, doc_root) return
def main(): """ parse command line options and call lib functions """ start_time = time.time() parser = argparse.ArgumentParser( description= 'Identify changes in current working directory as compared to a remote authoritative copy of the repo and identify all the elements affected by those changes.' ) # Note: I don't think we need to support files. If a file is submitted, CIS/QA can decompose it and then run this. So this can always run against repo. #parser.add_argument('-f', '--file', required=False, help='The name of the source file. If not used the local git repository will be used as the source') parser.add_argument('--silent', required=False, action="store_true", help='Suppress messages') parser.add_argument( '--remote', required=False, default='upstream', help="name of authoritative remote (default: 'upstream')") parser.add_argument( '--branch', required=False, default='master', help="name of branch in authoritative remote (default: 'master')") parser.add_argument( '--outfile', required=False, default='all.affected.oval.xml', help= "file name OVAL definitions file containing all affected definitions (default 'all.affected.oval.xml')" ) parser.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the affected definitions') args = vars(parser.parse_args()) silent = args['silent'] ## 1. Find Affected Elements # get changes in working dir vs. remote/branch message( 'info', 'Comparing working directory to {0}/{1}'.format( args['remote'], args['branch']), silent) paths_changed = lib_git.compare_current_oval_to_remote( args['remote'], args['branch']) if not paths_changed: message('info', 'No changes. Aborting.', silent) sys.exit(0) message( 'info', 'Found {0} files changed in working directory:\n\t{1}'.format( len(paths_changed), '\n\t'.join(paths_changed)), silent) # convert paths to oval ids oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in paths_changed } message( 'info', 'Found {0} OVAL elements changed in working directory:\n\t{1}'.format( len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent) # find all upstream ids message( 'info', 'Finding upstream OVAL ids for {0} element(s)'.format( len(oval_ids_changed)), silent) elements_index = lib_search.ElementsIndex(message) upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set()) message( 'info', 'Found {0} upstream OVAL ids (all element types)'.format( len(upstream_ids)), silent) affected_oval_ids = oval_ids_changed.union(upstream_ids) # filter affected to definition ids affected_def_ids = { oval_id for oval_id in upstream_ids if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition' } message( 'info', 'Found {0} upstream OVAL definitions:\n\t{1}'.format( len(affected_def_ids), '\n\t'.join(affected_def_ids)), silent) ## 2. Build an OVAL Definitions File and Validate It! message('info', 'Building an OVAL definitions file for all affected definitions.', silent) # get all downstream elements oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # add each OVAL definition to generator and write to file message( 'info', "Generating OVAL definition file '{0}' with {1} elements".format( args['outfile'], len(oval_ids)), silent) OvalGenerator = lib_xml.OvalGenerator(message) for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) OvalGenerator.to_file(args['outfile']) # validate schema_path = lib_repo.get_oval_def_schema('5.11.1') message('info', 'Performing schema validation', silent) try: lib_xml.schema_validate(args['outfile'], schema_path) message('info', 'Schema validation successful', silent) except lib_xml.SchemaValidationError as e: message('error', 'Schema validation failed:\n\t{0}'.format(e.message), silent) if args['schematron']: # schematron validate schema_path = lib_repo.get_oval_def_schema('5.11.1') message('info', 'Performing schematron validation', silent) try: lib_xml.schematron_validate(args['outfile'], schema_path) message('info', 'Schematron validation successful', silent) except lib_xml.SchematronValidationError as e: message( 'error', 'Schematron validation failed:\n\t{0}'.format( '\n\t'.join(e.messages)), silent) #Find all downstream children -- that is, a search depth of one #Find all upstream users, all the way up to the definition #Sort the list: definitions, then tests, objects, states, and variables #Show the list #Offer to build an OVAL file that contains all the changes seconds_elapsed = time.time() - start_time message('info', 'Completed in {0}!'.format(format_duration(seconds_elapsed)), silent)
def main(): start_time = time.time() tracking = dict() elements_index = lib_search.ElementsIndex(message) # getting all definitions all_def_ids = set() message('info', 'getting all definitions') for defpath in lib_repo.get_definition_paths_iterator(): all_def_ids.add(lib_repo.path_to_oval_id(defpath)) message('info', 'found {0} definitions'.format(len(all_def_ids))) # getting all element ids downstream from any definition message('info', 'getting all downstream element ids') all_downstream_ids = elements_index.find_downstream_ids(all_def_ids) message('info', 'found {0} downstream element ids'.format(len(all_downstream_ids))) # get elements that aren't in all_downstream_ids message('info', 'checking all elements') cur_element_type = None for elempath in lib_repo.get_element_paths_iterator(): oval_id = lib_repo.path_to_oval_id(elempath) element_type = lib_repo.get_element_type_from_oval_id(oval_id) # skip definitions... we're only pruning child elements if element_type == 'definition': continue # write status msg if element_type != cur_element_type: cur_element_type = element_type i_element_type = 0 i_element_type = i_element_type + 1 sys.stdout.write('Analyzing {0}s: {1} \r'.format(cur_element_type, i_element_type)) sys.stdout.flush() # it's an orphan if it's not downsteam of a definition track_as = 'orphan' if oval_id not in all_downstream_ids else 'in_use' if not track_as in tracking: tracking[track_as] = dict() if not element_type in tracking[track_as]: tracking[track_as][element_type] = set() tracking[track_as][element_type].add(oval_id) sys.stdout.write(' \r'.format(cur_element_type, i_element_type)) sys.stdout.flush() # generate report report = [] for track_as, elements_by_type in tracking.items(): report.append('\t{0}:'.format(track_as.replace('_', ' ').capitalize())) for element_type, oval_ids in elements_by_type.items(): report.append('\t\t{0} {1}s'.format(len(oval_ids), element_type)) message('found', '\n'.join(report)) response = input("\n :::: Remove all orphans? (N[o] / y[es]): ") if response.lower() == 'y': orphan_ids = set() for element_type, oval_ids in tracking['orphan'].items(): orphan_ids.update(oval_ids) file_paths = elements_index.get_paths_from_ids(orphan_ids) for file_path in file_paths: message("INFO", "Deleting Orphan '%s'" % os.path.basename(file_path)) os.remove(file_path) seconds_elapsed = time.time() - start_time message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))