def main(): today = datetime.date.today() url = r"http://oval.mitre.org/rep-data/5.10/org.mitre.oval/oval.xml" dest = r"C:\_Development\oval_repository\Serializer\processed\oval-%s-%s-%s.xml" % ( today.year, today.month, today.day) (fn, msg) = urllib.request.urlretrieve(url, dest) message("INFO", "Downloaded file %s" % fn) oval_decomposition.decompose(fn, True) # get indexes definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) for defpath in lib_repo.get_definition_paths_iterator(): message("INFO", "Determining minimum schema version for file %s" % defpath) result = lib_schema.determine_def_min_version(defpath, definitions_index, elements_index, True) message( "INFO", "The minimum OVAL version supported by '{0}' is {1}.".format( defpath, result['minimum_version']))
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser(description='Determine minimum supported OVAL schema for a given definition') parser.add_argument('-p', '--path', help='path to definition fragment') parser.add_argument('-i', '--id', help='id of definition fragment') parser.add_argument('-a', '--all', default=False, action="store_true", help='determine minimum supported OVAL schema for all indexed definition fragments. cannot be combined with --id or --path') parser.add_argument('-u', '--update', default=False, action="store_true", help='update the given definition to include the min_schema_version element') args = vars(parser.parse_args()) if (args['all'] and (args['id'] or args['path'])): parser.print_help() message('error', "--all cannot be combined with --id or --path.") sys.exit(0) if not args['all'] and not (args['path'] or args['id']): parser.print_help() message('error',"Please specify one of: --all, --path or --id.") sys.exit(0) if (args['path'] and args['id']): parser.print_help() message('error',"Please specify one of: --all, --path, or --id.") sys.exit(0) # get indexes definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) if args['all']: for defpath in lib_repo.get_definition_paths_iterator(): result = determine_def_min_version(defpath, definitions_index, elements_index, args['update']) report(defpath, result['minimum_version']) elif args['id']: documents = definitions_index.query({ 'oval_id': args['id'] }) if len(documents) == 0: message('info',"No definitions having id '{0}' found.".format(args['id'])) sys.exit(0) paths = [ document['path'] for document in documents ] for path in paths: result = determine_def_min_version(path, definitions_index, elements_index, args['update']) report(result['oval_id'], result['minimum_version']) elif args['path'] and not os.path.isfile(args['path']): message('info',"Definition fragment path not found: " + args['path']) sys.exit(0) else: result = determine_def_min_version(args['path'], definitions_index, elements_index, args['update']) report(args['path'], result['minimum_version']) seconds_elapsed = time.time() - start_time report('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser( description='Builds all OVAL definitons in the repository.') parser.add_argument( '-l', '--limit', nargs='?', default="0", type=int, help='limits number of definitions that will be built)') args = vars(parser.parse_args()) # get indexes definitions_index = lib_search.DefinitionsIndex(message) elements_index = lib_search.ElementsIndex(message) # create generator, build in memory b/c smallish files OvalGenerator = lib_xml.OvalGenerator(message) OvalGenerator.use_file_queues = False i_file = 0 i_limit = args['limit'] for defpath in lib_repo.get_definition_paths_iterator(): i_file = i_file + 1 if i_limit > 0 and i_file > i_limit: break def_id = lib_repo.path_to_oval_id(defpath) message('info', 'Building file {0} for {1}.'.format(i_file, def_id)) # add all downstream element ids def_ids = set([def_id]) oval_ids = elements_index.find_downstream_ids(def_ids, def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # add each OVAL definition to generator for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) # write output file outfile = './tmp/gen.{0}'.format(lib_repo.oval_id_to_path(def_id)) OvalGenerator.to_file(outfile) seconds_elapsed = time.time() - start_time message('info', 'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def get_definition_path(self): oval_query = {'reference_ids': self.cve_id} query_results = lib_search.DefinitionsIndex().query(oval_query) if not query_results: return False # TODO support other classes for item in query_results: if item['class'] == 'vulnerability': self.xml_path = item['path'] else: continue return True
def main(): query = {} definitions_index = lib_search.DefinitionsIndex(message) #revisions_index = lib_search.RevisionsIndex(message) #filtered_oval_ids = revisions_index.get_definition_ids({ 'date': revisions_index.format_daterange("20151201", "20151231") }) #filtered_oval_ids = revisions_index.get_definition_ids({ 'status': 'INITIAL SUBMISSION' }) #for ovalid in filtered_oval_ids: # message("INFO", "Filtered oval id is %s" % ovalid) query["status"] = "INITIAL SUBMISSION" #query['oval_id'] = filtered_oval_ids query_results = definitions_index.query(query) #definition_ids = { document['oval_id'] for document in query_results } for document in query_results: message("INFO", "OVAL ID %s has status %s" % (document["oval_id"], document["status"]))
def main(): start_time = time.time() definitions_index = lib_search.DefinitionsIndex(message) ns_map = {'oval-def': 'http://oval.mitre.org/XMLSchema/oval-definitions-5'} # Create the output object... output = WebOutput() for defpath in lib_repo.get_definition_paths_iterator(): def_id = lib_repo.path_to_oval_id(defpath) try: tree = etree.parse(defpath) except OSError as e: raise InvalidPathError(defpath) except etree.XMLSyntaxError as e: raise InvalidXmlError(defpath, e.msg) root = tree.getroot() # Only Vulnerability and Patch definitions would have CVE references... defclass = root.get('class') if defclass in ["vulnerability", "patch"]: reference = root.find( "./oval-def:metadata/oval-def:reference[@source='CVE']", ns_map) if reference is not None: output.add_result( def_id, [reference.get("ref_id"), reference.get("ref_url")]) # add timing seconds_elapsed = time.time() - start_time output.message("CVE", "OVAL Repository Definition-to-CVE Mappings") output.message("time", "{0}".format(format_duration(seconds_elapsed))) # Write JSON Output... output.write_json()
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser( description='Builds a schema-valid OVAL definitions file.') output_options = parser.add_argument_group('output options') output_options.add_argument( '-o', '--outfile', required=True, help='file name for output OVAL definitions file') output_options.add_argument('-v', '--validate', default=False, action="store_true", help='schema validate the output file') output_options.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the output file') output_options.add_argument( '-t', '--tempdir', required=False, default="./", help= "directory to store temporary files used when building oval definitions (default: './')" ) source_options = parser.add_argument_group( 'definitions filtering', 'Provide at least one of the following options to determine which definition(s) ' + 'will be included. Results will include the intersection of matches for each parameter ' + 'supplied. When multiple values are supplied for one paramater, the parameter will ' + 'match definitions that match any provided value.') source_options.add_argument('--definition_id', nargs='*', dest='oval_id', help='match OVAL definition id(s)') source_options.add_argument('--title', nargs='*', dest='title', metavar='PHRASE', help='match phrase(s) in definition titles') source_options.add_argument('--description', nargs='*', dest='description', metavar='PHRASE', help='match phrase(s) in definition titles') source_options.add_argument( '--class', nargs='*', dest='class', help='filter by class(es): {0}'.format(', '.join( lib_repo.supported_definition_classes))) source_options.add_argument( '--status', nargs='*', dest='status', help='filter by status(es): {0}'.format(', '.join( lib_repo.supported_definition_statuses))) source_options.add_argument('--family', nargs='*', dest='family', help='filter by family(ies)') source_options.add_argument('--platform', nargs='*', dest='platforms', metavar='PLATFORM', help='filter by platform(s)') source_options.add_argument('--product', nargs='*', dest='products', metavar='PRODUCT', help='filter by product(s)') source_options.add_argument('--contributor', nargs='*', dest='contributors', metavar='NAME', help='filter by contributor(s)') source_options.add_argument('--organization', nargs='*', dest='organizations', metavar='NAME', help='filter by organization(s)') source_options.add_argument( '--reference_id', nargs='*', dest='reference_ids', metavar='REFERENCE_ID', help='filter by reference ids, e.g. CVE-2015-3306') source_options.add_argument( '--max_schema_version', nargs="?", dest='max_schema_version', metavar='SCHEMA_VERSION', help='filter by maximum oval schema version, e.g. 5.10') source_options.add_argument( '--all_definitions', default=False, action="store_true", help= 'include all definitions in the repository (do not specify any other filters)' ) source_options.add_argument( '--from', nargs='?', default='', metavar='YYYYMMDD', help='include elements revised on or after this day (format: YYYYMMDD)' ) source_options.add_argument( '--to', nargs='?', default='', metavar='YYYYMMDD', help='include elements revised on or before this day (format: YYYYMMDD)' ) args = vars(parser.parse_args()) # get definitions index definitions_index = lib_search.DefinitionsIndex(message) # contruct query from args query = {} for field in definitions_index.get_fieldnames(): if field in args and args[field]: query[field] = args[field] # add schema_version filter, if specified if args['max_schema_version']: query['min_schema_version'] = '[0 TO {0}]'.format( definitions_index.version_to_int(args['max_schema_version'])) # add date range and contributor/org filters, if specified all_definitions_filtered = False if args['from'] or args['to'] or args['contributors'] or args[ 'organizations']: # get revisions index revisions_index = lib_search.RevisionsIndex(message) if args['from'] or args['to']: filtered_oval_ids = revisions_index.get_definition_ids({ 'date': revisions_index.format_daterange(args['from'], args['to']) }) if args['contributors']: contributor_filtered_ids = revisions_index.get_definition_ids( {'contributor': args['contributors']}) filtered_oval_ids = filtered_oval_ids & contributor_filtered_ids if 'filtered_oval_ids' in locals( ) else contributor_filtered_ids if args['organizations']: organization_filtered_ids = revisions_index.get_definition_ids( {'organization': args['organizations']}) filtered_oval_ids = filtered_oval_ids & organization_filtered_ids if 'filtered_oval_ids' in locals( ) else organization_filtered_ids # add to query if 'oval_id' in query and query['oval_id']: # if oval_id(s) specified in args, get intersection with filtered oval ids query['oval_id'] = set(query['oval_id']) & filtered_oval_ids else: query['oval_id'] = filtered_oval_ids if not query['oval_id']: all_definitions_filtered = True # --all_definitions OR at least one definition selection option must be specified if args['all_definitions'] and query: parser.print_help() message( 'error', "The '--all_definitions' filter cannot be combined with any other filters." ) sys.exit(1) elif not (args['all_definitions'] or query): parser.print_help() message( 'error', 'At least one definitions filtering argument must be provided.') sys.exit(1) # query index query_results = definitions_index.query( query) if not all_definitions_filtered else {} # get set of all definition ids found definition_ids = {document['oval_id'] for document in query_results} message('info', 'Found {0} matching OVAL definitions'.format(len(definition_ids))) # create generator and set oval schema version, if necessary OvalGenerator = lib_xml.OvalGenerator(message, args['tempdir']) if args['max_schema_version']: OvalGenerator.oval_schema_version = args['max_schema_version'] if definition_ids: # add all downstream element ids message('info', 'Finding downstream OVAL ids for all definitions') elements_index = lib_search.ElementsIndex(message) oval_ids = elements_index.find_downstream_ids(definition_ids, definition_ids) message( 'info', 'Found {0} downstream OVAL ids'.format( len(oval_ids) - len(query_results))) # get paths for all elements message('info', 'Finding paths for {0} OVAL elements'.format(len(oval_ids))) file_paths = elements_index.get_paths_from_ids(oval_ids) # build in memory if there aren't that many files if len(file_paths) < 200: OvalGenerator.use_file_queues = False # add each OVAL definition to generator message( 'info', 'Generating OVAL definition file with {0} elements'.format( len(oval_ids))) for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) # write output file message('info', 'Writing OVAL definitions to {0}'.format(args['outfile'])) OvalGenerator.to_file(args['outfile']) # validate if args['validate']: # schema validate schema_path = lib_repo.get_oval_def_schema( OvalGenerator.oval_schema_version) message('info', 'performing schema validation') try: lib_xml.schema_validate(args['outfile'], schema_path) message('info', 'schema validation successful') except lib_xml.SchemaValidationError as e: message('error', 'schema validation failed:\n\t{0}'.format(e.message)) if args['schematron']: # schematron validate schema_path = lib_repo.get_oval_def_schema( OvalGenerator.oval_schema_version) message('info', 'performing schematron validation') try: lib_xml.schematron_validate(args['outfile'], schema_path) message('info', 'schematron validation successful') except lib_xml.SchematronValidationError as e: message( 'error', 'schematron validation failed:\n\t{0}'.format( '\n\t'.join(e.messages))) seconds_elapsed = time.time() - start_time message('info', 'Completed in {0}!'.format(format_duration(seconds_elapsed)))