def main(): today = datetime.date.today() url = r"http://oval.mitre.org/rep-data/5.10/org.mitre.oval/oval.xml" dest = r"C:\_Development\oval_repository\Serializer\processed\oval-%s-%s-%s.xml" % ( today.year, today.month, today.day) (fn, msg) = urllib.request.urlretrieve(url, dest) message("INFO", "Downloaded file %s" % fn) oval_decomposition.decompose(fn, True) # get indexes definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) for defpath in lib_repo.get_definition_paths_iterator(): message("INFO", "Determining minimum schema version for file %s" % defpath) result = lib_schema.determine_def_min_version(defpath, definitions_index, elements_index, True) message( "INFO", "The minimum OVAL version supported by '{0}' is {1}.".format( defpath, result['minimum_version']))
def main(): start_time = time.time() definitions_index = lib_search.DefinitionsIndex(message) ns_map = {"oval-def": "http://oval.mitre.org/XMLSchema/oval-definitions-5"} # Create the output object... output = WebOutput() for defpath in lib_repo.get_definition_paths_iterator(): def_id = lib_repo.path_to_oval_id(defpath) try: tree = etree.parse(defpath) except OSError as e: raise InvalidPathError(defpath) except etree.XMLSyntaxError as e: raise InvalidXmlError(defpath, e.msg) root = tree.getroot() # Only Vulnerability and Patch definitions would have CVE references... defclass = root.get("class") if defclass in ["vulnerability", "patch"]: reference = root.find("./oval-def:metadata/oval-def:reference[@source='CVE']", ns_map) if reference is not None: output.add_result(def_id, [reference.get("ref_id"), reference.get("ref_url")]) # add timing seconds_elapsed = time.time() - start_time output.message("CVE", "OVAL Repository Definition-to-CVE Mappings") output.message("time", "{0}".format(format_duration(seconds_elapsed))) # Write JSON Output... output.write_json()
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser(description='Determine minimum supported OVAL schema for a given definition') parser.add_argument('-p', '--path', help='path to definition fragment') parser.add_argument('-i', '--id', help='id of definition fragment') parser.add_argument('-a', '--all', default=False, action="store_true", help='determine minimum supported OVAL schema for all indexed definition fragments. cannot be combined with --id or --path') parser.add_argument('-u', '--update', default=False, action="store_true", help='update the given definition to include the min_schema_version element') args = vars(parser.parse_args()) if (args['all'] and (args['id'] or args['path'])): parser.print_help() message('error', "--all cannot be combined with --id or --path.") sys.exit(0) if not args['all'] and not (args['path'] or args['id']): parser.print_help() message('error',"Please specify one of: --all, --path or --id.") sys.exit(0) if (args['path'] and args['id']): parser.print_help() message('error',"Please specify one of: --all, --path, or --id.") sys.exit(0) # get indexes definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) if args['all']: for defpath in lib_repo.get_definition_paths_iterator(): result = determine_def_min_version(defpath, definitions_index, elements_index, args['update']) report(defpath, result['minimum_version']) elif args['id']: documents = definitions_index.query({ 'oval_id': args['id'] }) if len(documents) == 0: message('info',"No definitions having id '{0}' found.".format(args['id'])) sys.exit(0) paths = [ document['path'] for document in documents ] for path in paths: result = determine_def_min_version(path, definitions_index, elements_index, args['update']) report(result['oval_id'], result['minimum_version']) elif args['path'] and not os.path.isfile(args['path']): message('info',"Definition fragment path not found: " + args['path']) sys.exit(0) else: result = determine_def_min_version(args['path'], definitions_index, elements_index, args['update']) report(args['path'], result['minimum_version']) seconds_elapsed = time.time() - start_time report('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser( description='Builds all OVAL definitons in the repository.') parser.add_argument( '-l', '--limit', nargs='?', default="0", type=int, help='limits number of definitions that will be built)') args = vars(parser.parse_args()) # get indexes # definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) # create generator, build in memory b/c smallish files OvalGenerator = lib_xml.OvalGenerator(message) OvalGenerator.use_file_queues = False print("Storing data in temp directory: {}".format(tempfile.gettempdir())) i_file = 0 i_limit = args['limit'] for defpath in lib_repo.get_definition_paths_iterator(): i_file = i_file + 1 if i_limit > 0 and i_file > i_limit: break def_id = lib_repo.path_to_oval_id(defpath) message('info', 'Building file {0} for {1}.'.format(i_file, def_id)) # add all downstream element ids def_ids = set([def_id]) oval_ids = elements_index.find_downstream_ids(def_ids, def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # add each OVAL definition to generator for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) # write output file outfile = '{1}/gen.{0}'.format(lib_repo.oval_id_to_path(def_id), tempfile.gettempdir()) OvalGenerator.to_file(outfile) seconds_elapsed = time.time() - start_time message('info', 'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def document_iterator(self, paths=False): """ Iterator yielding definition files in repo as indexable documents. """ if not paths: # get all in repo paths = lib_repo.get_definition_paths_iterator() for path in paths: try: document = lib_xml.get_definition_metadata(path) except lib_xml.InvalidPathError as e: yield { 'path': e.path, 'deleted': True } else: document = self.whoosh_escape_document(document) yield document
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser(description="Builds all OVAL definitons in the repository.") parser.add_argument( "-l", "--limit", nargs="?", default="0", type=int, help="limits number of definitions that will be built)" ) args = vars(parser.parse_args()) # get indexes definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) # create generator, build in memory b/c smallish files OvalGenerator = lib_xml.OvalGenerator(message) OvalGenerator.use_file_queues = False i_file = 0 i_limit = args["limit"] for defpath in lib_repo.get_definition_paths_iterator(): i_file = i_file + 1 if i_limit > 0 and i_file > i_limit: break def_id = lib_repo.path_to_oval_id(defpath) message("info", "Building file {0} for {1}.".format(i_file, def_id)) # add all downstream element ids def_ids = set([def_id]) oval_ids = elements_index.find_downstream_ids(def_ids, def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # add each OVAL definition to generator for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) # write output file outfile = "./tmp/gen.{0}".format(lib_repo.oval_id_to_path(def_id)) OvalGenerator.to_file(outfile) seconds_elapsed = time.time() - start_time message("info", "Completed in {0}!".format(format_duration(seconds_elapsed)))
def document_iterator(self, paths=False): """ Iterator yielding definition files in repo as indexable documents. """ if not paths: # get all in repo paths = lib_repo.get_definition_paths_iterator() for path in paths: try: document = lib_xml.get_definition_metadata(path) except lib_xml.InvalidPathError as e: yield { 'path': e.path, 'deleted': True } except lib_xml.InvalidXmlError as e: raise else: del document['revisions'] document = self.whoosh_escape_document(document) document['min_schema_version'] = self.version_to_int(document['min_schema_version']) yield document
def document_iterator(self, paths=False): """ Iterator yielding definition revisions in repo as indexable documents. """ if not paths: # get all in repo paths = lib_repo.get_definition_paths_iterator() for path in paths: try: document = lib_xml.get_definition_metadata(path) except lib_xml.InvalidPathError as e: yield { 'oval_id': lib_repo.path_to_oval_id(e.path), 'deleted': True } except lib_xml.InvalidXmlError as e: raise else: oval_id = document['oval_id'] for revision in document['revisions']: revision['oval_id'] = oval_id revision = self.whoosh_escape_document(revision) yield revision
def main(): today = datetime.date.today() url = r"http://oval.mitre.org/rep-data/5.10/org.mitre.oval/oval.xml" dest = r"C:\_Development\oval_repository\Serializer\processed\oval-%s-%s-%s.xml" % (today.year, today.month, today.day) (fn, msg) = urllib.request.urlretrieve(url, dest) message("INFO", "Downloaded file %s" % fn) oval_decomposition.decompose(fn, True) # get indexes definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) for defpath in lib_repo.get_definition_paths_iterator(): message("INFO", "Determining minimum schema version for file %s" % defpath) result = lib_schema.determine_def_min_version(defpath, definitions_index, elements_index, True) message("INFO", "The minimum OVAL version supported by '{0}' is {1}.".format(defpath, result['minimum_version']))
def main(): start_time = time.time() definitions_index = lib_search.DefinitionsIndex(message) ns_map = {'oval-def': 'http://oval.mitre.org/XMLSchema/oval-definitions-5'} # Create the output object... output = WebOutput() for defpath in lib_repo.get_definition_paths_iterator(): def_id = lib_repo.path_to_oval_id(defpath) try: tree = etree.parse(defpath) except OSError as e: raise InvalidPathError(defpath) except etree.XMLSyntaxError as e: raise InvalidXmlError(defpath, e.msg) root = tree.getroot() # Only Vulnerability and Patch definitions would have CVE references... defclass = root.get('class') if defclass in ["vulnerability", "patch"]: reference = root.find( "./oval-def:metadata/oval-def:reference[@source='CVE']", ns_map) if reference is not None: output.add_result( def_id, [reference.get("ref_id"), reference.get("ref_url")]) # add timing seconds_elapsed = time.time() - start_time output.message("CVE", "OVAL Repository Definition-to-CVE Mappings") output.message("time", "{0}".format(format_duration(seconds_elapsed))) # Write JSON Output... output.write_json()
def main(): start_time = time.time() tracking = dict() elements_index = lib_search.ElementsIndex(message) # getting all definitions all_def_ids = set() message('info', 'getting all definitions') for defpath in lib_repo.get_definition_paths_iterator(): all_def_ids.add(lib_repo.path_to_oval_id(defpath)) message('info', 'found {0} definitions'.format(len(all_def_ids))) # getting all element ids downstream from any definition message('info', 'getting all downstream element ids') all_downstream_ids = elements_index.find_downstream_ids(all_def_ids) message('info', 'found {0} downstream element ids'.format(len(all_downstream_ids))) # get elements that aren't in all_downstream_ids message('info', 'checking all elements') cur_element_type = None for elempath in lib_repo.get_element_paths_iterator(): oval_id = lib_repo.path_to_oval_id(elempath) element_type = lib_repo.get_element_type_from_oval_id(oval_id) # skip definitions... we're only pruning child elements if element_type == 'definition': continue # write status msg if element_type != cur_element_type: cur_element_type = element_type i_element_type = 0 i_element_type = i_element_type + 1 sys.stdout.write( 'Analyzing {0}s: {1} \r'.format( cur_element_type, i_element_type)) sys.stdout.flush() # it's an orphan if it's not downsteam of a definition track_as = 'orphan' if oval_id not in all_downstream_ids else 'in_use' if not track_as in tracking: tracking[track_as] = dict() if not element_type in tracking[track_as]: tracking[track_as][element_type] = set() tracking[track_as][element_type].add(oval_id) sys.stdout.write( ' \r'. format(cur_element_type, i_element_type)) sys.stdout.flush() # generate report report = [] for track_as, elements_by_type in tracking.items(): report.append('\t{0}:'.format(track_as.replace('_', ' ').capitalize())) for element_type, oval_ids in elements_by_type.items(): report.append('\t\t{0} {1}s'.format(len(oval_ids), element_type)) message('found', '\n'.join(report)) response = input("\n :::: Remove all orphans? (N[o] / y[es]): ") if response.lower() == 'y': orphan_ids = set() for element_type, oval_ids in tracking['orphan'].items(): orphan_ids.update(oval_ids) file_paths = elements_index.get_paths_from_ids(orphan_ids) for file_path in file_paths: message("INFO", "Deleting Orphan '%s'" % os.path.basename(file_path)) os.remove(file_path) seconds_elapsed = time.time() - start_time message('info', 'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main(): start_time = time.time() tracking = dict() elements_index = lib_search.ElementsIndex(message) # getting all definitions all_def_ids = set() message('info', 'getting all definitions') for defpath in lib_repo.get_definition_paths_iterator(): all_def_ids.add(lib_repo.path_to_oval_id(defpath)) message('info', 'found {0} definitions'.format(len(all_def_ids))) # getting all element ids downstream from any definition message('info', 'getting all downstream element ids') all_downstream_ids = elements_index.find_downstream_ids(all_def_ids) message('info', 'found {0} downstream element ids'.format(len(all_downstream_ids))) # get elements that aren't in all_downstream_ids message('info', 'checking all elements') cur_element_type = None for elempath in lib_repo.get_element_paths_iterator(): oval_id = lib_repo.path_to_oval_id(elempath) element_type = lib_repo.get_element_type_from_oval_id(oval_id) # skip definitions... we're only pruning child elements if element_type == 'definition': continue # write status msg if element_type != cur_element_type: cur_element_type = element_type i_element_type = 0 i_element_type = i_element_type + 1 sys.stdout.write('Analyzing {0}s: {1} \r'.format(cur_element_type, i_element_type)) sys.stdout.flush() # it's an orphan if it's not downsteam of a definition track_as = 'orphan' if oval_id not in all_downstream_ids else 'in_use' if not track_as in tracking: tracking[track_as] = dict() if not element_type in tracking[track_as]: tracking[track_as][element_type] = set() tracking[track_as][element_type].add(oval_id) sys.stdout.write(' \r'.format(cur_element_type, i_element_type)) sys.stdout.flush() # generate report report = [] for track_as, elements_by_type in tracking.items(): report.append('\t{0}:'.format(track_as.replace('_', ' ').capitalize())) for element_type, oval_ids in elements_by_type.items(): report.append('\t\t{0} {1}s'.format(len(oval_ids), element_type)) message('found', '\n'.join(report)) response = input("\n :::: Remove all orphans? (N[o] / y[es]): ") if response.lower() == 'y': orphan_ids = set() for element_type, oval_ids in tracking['orphan'].items(): orphan_ids.update(oval_ids) file_paths = elements_index.get_paths_from_ids(orphan_ids) for file_path in file_paths: message("INFO", "Deleting Orphan '%s'" % os.path.basename(file_path)) os.remove(file_path) seconds_elapsed = time.time() - start_time message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser(description="Determine minimum supported OVAL schema for a given definition") parser.add_argument("-p", "--path", help="path to definition fragment") parser.add_argument("-i", "--id", help="id of definition fragment") parser.add_argument( "-a", "--all", default=False, action="store_true", help="determine minimum supported OVAL schema for all indexed definitino fragments. cannot be combined with --id or --path", ) parser.add_argument( "-u", "--update", default=False, action="store_true", help="update the given definition to include the min_schema_version element", ) args = vars(parser.parse_args()) if args["all"] and (args["id"] or args["path"]): parser.print_help() message("error", "--all cannot be combined with --id or --path.") sys.exit(0) if not args["all"] and not (args["path"] or args["id"]): parser.print_help() message("error", "Please specify one of: --all, --path or --id.") sys.exit(0) if args["path"] and args["id"]: parser.print_help() message("error", "Please specify one of: --all, --path, or --id.") sys.exit(0) # get indexes definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) # build schema path cache for schema_version in lib_repo.get_schema_versions(): schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version) if args["all"]: for defpath in lib_repo.get_definition_paths_iterator(): result = determine_def_min_version(defpath, definitions_index, elements_index, args["update"]) report(defpath, result["minimum_version"]) elif args["id"]: documents = definitions_index.query({"oval_id": args["id"]}) if len(documents) == 0: message("info", "No definitions having id '{0}' found.".format(args["id"])) sys.exit(0) paths = [document["path"] for document in documents] for path in paths: result = determine_def_min_version(path, definitions_index, elements_index, args["update"]) report(result["oval_id"], result["minimum_version"]) elif args["path"] and not os.path.isfile(args["path"]): message("info", "Definition fragment path not found: " + args["path"]) sys.exit(0) else: result = determine_def_min_version(args["path"], definitions_index, elements_index, args["update"]) report(args["path"], result["minimum_version"]) seconds_elapsed = time.time() - start_time report("info", "Completed in {0}!".format(format_duration(seconds_elapsed)))