def main(): """ parse command line options and generate statistics """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser( description='Get repository content statistics for website.') groups_options = parser.add_argument_group( 'groups', 'Specify one or more groups to only show those groups. Displays all by default.' ) for field in grouping_fields.keys(): label = grouping_fields[field]['label'] groups_options.add_argument('--{0}'.format(label), default=False, action="store_true", help='show totals by {0}'.format(label)) args = vars(parser.parse_args()) # get list of groups specified in args show_grouping_fields = [ group for group in grouping_fields.keys() if args[grouping_fields[group]['label']] ] # if not groups specified, show all if not show_grouping_fields: show_grouping_fields = grouping_fields.keys() # get output object output = WebOutput() # get index references index_refs = { 'definitions': lib_search.ThreadSafeDefinitionsIndex(output.message), 'revisions': lib_search.ThreadSafeRevisionsIndex(output.message), 'elements': lib_search.ThreadSafeElementsIndex(output.message) } for index in index_refs: index_refs[index].no_output = True # output the groups we're showing total_groups = len(show_grouping_fields) for i, group in enumerate(show_grouping_fields): index = index_refs[grouping_fields[group]['index']] group_totals = get_grouped_totals(group, index) output.add_result(group, group_totals) # add timinig seconds_elapsed = time.time() - start_time output.message('time', '{0}'.format(format_duration(seconds_elapsed))) # output results output.write_json()
def main(): """ parse command line options and generate statistics """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser( description='Get repository contributor statistics for website.') filter_options = parser.add_argument_group('filter options') filter_options.add_argument( '--from', nargs='?', default='', metavar='YYYYMMDD', help='omit contributions before this day (format: YYYYMMDD)') filter_options.add_argument( '--to', nargs='?', default='', metavar='YYYYMMDD', help='omit contributions after this day (format: YYYYMMDD)') args = vars(parser.parse_args()) # get output object output = WebOutput() # get index revisions_index = lib_search.ThreadSafeRevisionsIndex(output.message) revisions_index.no_output = True # construct query if args['from'] or args['to']: date_range = "{0} TO {1}".format(args['from'] or '', args['to'] or '').strip() query = {"date": '[{0}]'.format(date_range)} else: query = {} # output the groups we're showing results = revisions_index.grouped_query(query, ['contributor', 'type']) output.add_result('contributor', format_grouped_totals(results)) results = revisions_index.grouped_query(query, ['organization', 'type']) output.add_result('organization', format_grouped_totals(results)) # add timinig seconds_elapsed = time.time() - start_time output.message('time', '{0}'.format(format_duration(seconds_elapsed))) # output results output.write_json()
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser( description= 'Updates indexes. Should be run one each time new OVAL content is published.' ) parser.add_argument('--nocache', required=False, action="store_true", help='Rebuild from scratch even if not necessary') args = vars(parser.parse_args()) # get output object output = WebOutput() # get definitions index and update index = lib_search.ThreadSafeDefinitionsIndex(output.message) index.no_output = True index.update(args['nocache']) output.message('definitions', 'index updated') # get elements index and update index = lib_search.ThreadSafeElementsIndex(output.message) index.no_output = True index.update(args['nocache']) output.message('elements', 'index updated') # get elements index and update index = lib_search.ThreadSafeRevisionsIndex(output.message) index.no_output = True index.update(args['nocache']) output.message('revisions', 'index updated') # add timinig seconds_elapsed = time.time() - start_time output.message('time', '{0}'.format(format_duration(seconds_elapsed))) # output results output.write_json()
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser( description= 'Powers website-intiated builds if schema-valid OVAL definitions file. Note: unlike build_oval_definitions_file.py, this script will not automatically build/update the indices. You must run web_update_indexes.py before running this script to ensure you get accurate results.' ) output_options = parser.add_argument_group('output options') output_options.add_argument( '-o', '--outfile', required=True, help='file name for output OVAL definitions file') output_options.add_argument('-v', '--validate', default=False, action="store_true", help='schema validate the output file') output_options.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the output file') output_options.add_argument( '-t', '--tempdir', required=False, default="./", help= "directory to store temporary files used when building oval definitions (default: './')" ) source_options = parser.add_argument_group( 'definitions filtering', 'Provide at least one of the following options to determine which definition(s) ' + 'will be included. Results will include the intersection of matches for each parameter ' + 'supplied. When multiple values are supplied for one paramater, the parameter will ' + 'match definitions that match any provided value.') source_options.add_argument('--definition_id', nargs='*', dest='oval_id', help='match OVAL definition id(s)') source_options.add_argument('--title', nargs='*', dest='title', metavar='PHRASE', help='match phrase(s) in definition titles') source_options.add_argument('--description', nargs='*', dest='description', metavar='PHRASE', help='match phrase(s) in definition titles') source_options.add_argument( '--class', nargs='*', dest='class', help='filter by class(es): {0}'.format(', '.join( lib_repo.supported_definition_classes))) source_options.add_argument( '--status', nargs='*', dest='status', help='filter by status(es): {0}'.format(', '.join( lib_repo.supported_definition_statuses))) source_options.add_argument('--family', nargs='*', dest='family', help='filter by family(ies)') source_options.add_argument('--platform', nargs='*', dest='platforms', metavar='PLATFORM', help='filter by platform(s)') source_options.add_argument('--product', nargs='*', dest='products', metavar='PRODUCT', help='filter by product(s)') source_options.add_argument('--contributor', nargs='*', dest='contributors', metavar='NAME', help='filter by contributor(s)') source_options.add_argument('--organization', nargs='*', dest='organizations', metavar='NAME', help='filter by organization(s)') source_options.add_argument( '--reference_id', nargs='*', dest='reference_ids', metavar='REFERENCE_ID', help='filter by reference ids, e.g. CVE-2015-3306') source_options.add_argument( '--max_schema_version', nargs="?", dest='max_schema_version', metavar='SCHEMA_VERSION', help='filter by maximum oval schema version, e.g. 5.10') source_options.add_argument( '--all_definitions', default=False, action="store_true", help= 'include all definitions in the repository (do not specify any other filters)' ) source_options.add_argument( '--from', nargs='?', default='', metavar='YYYYMMDD', help='include elements revised on or after this day (format: YYYYMMDD)' ) source_options.add_argument( '--to', nargs='?', default='', metavar='YYYYMMDD', help='include elements revised on or before this day (format: YYYYMMDD)' ) args = vars(parser.parse_args()) # get output object output = WebOutput() # get index definitions_index = lib_search.ThreadSafeDefinitionsIndex(output.message) definitions_index.no_output = True # contruct query from args query = {} for field in definitions_index.get_fieldnames(): if field in args and args[field]: query[field] = args[field] # add schema_version filter, if specified if args['max_schema_version']: query['min_schema_version'] = '[0 TO {0}]'.format( definitions_index.version_to_int(args['max_schema_version'])) # add date range and contributor/org filters, if specified all_definitions_filtered = False if args['from'] or args['to'] or args['contributors'] or args[ 'organizations']: # get revisions index revisions_index = lib_search.ThreadSafeRevisionsIndex(output.message) revisions_index.no_output = True if args['from'] or args['to']: filtered_oval_ids = revisions_index.get_definition_ids({ 'date': revisions_index.format_daterange(args['from'], args['to']) }) if args['contributors']: contributor_filtered_ids = revisions_index.get_definition_ids( {'contributor': args['contributors']}) filtered_oval_ids = filtered_oval_ids & contributor_filtered_ids if 'filtered_oval_ids' in locals( ) else contributor_filtered_ids if args['organizations']: organization_filtered_ids = revisions_index.get_definition_ids( {'organization': args['organizations']}) filtered_oval_ids = filtered_oval_ids & organization_filtered_ids if 'filtered_oval_ids' in locals( ) else organization_filtered_ids # add to query if 'oval_id' in query and query['oval_id']: # if oval_id(s) specified in args, get intersection with filtered oval ids query['oval_id'] = set(query['oval_id']) & filtered_oval_ids else: query['oval_id'] = filtered_oval_ids if not query['oval_id']: all_definitions_filtered = True # --all_definitions OR at least one definition selection option must be specified if args['all_definitions'] and query: parser.print_help() output.message( 'error', "The '--all_definitions' filter cannot be combined with any other filters." ) sys.exit(1) elif not (args['all_definitions'] or query): parser.print_help() output.message( 'error', 'At least one definitions filtering argument must be provided.') sys.exit(1) # query index query_results = definitions_index.query( query) if not all_definitions_filtered else {} # get set of all definition ids found definition_ids = {document['oval_id'] for document in query_results} output.message( 'info', 'Found {0} matching OVAL definitions'.format(len(definition_ids))) # create generator and set oval schema version, if necessary OvalGenerator = lib_xml.OvalGenerator(output.message, args['tempdir']) if args['max_schema_version']: OvalGenerator.oval_schema_version = args['max_schema_version'] if definition_ids: # add all downstream element ids output.message('info', 'Finding downstream OVAL ids for all definitions') elements_index = lib_search.ThreadSafeElementsIndex(output.message) elements_index.no_output = True oval_ids = elements_index.find_downstream_ids(definition_ids, definition_ids) output.message( 'info', 'Found {0} downstream OVAL ids'.format( len(oval_ids) - len(query_results))) # get paths for all elements output.message( 'info', 'Finding paths for {0} OVAL elements'.format(len(oval_ids))) file_paths = elements_index.get_paths_from_ids(oval_ids) # build in memory if there aren't that many files if len(file_paths) < 200: OvalGenerator.use_file_queues = False # add each OVAL definition to generator output.message( 'info', 'Generating OVAL definition file with {0} elements'.format( len(oval_ids))) for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) # write output file output.message('info', 'Writing OVAL definitions to {0}'.format(args['outfile'])) OvalGenerator.to_file(args['outfile']) output.add_result(args['outfile']) # validate if args['validate']: # schema validate schema_path = lib_repo.get_oval_def_schema( OvalGenerator.oval_schema_version) output.message('info', 'performing schema validation') try: lib_xml.schema_validate(args['outfile'], schema_path) output.message('info', 'schema validation successful') except lib_xml.SchemaValidationError as e: output.message( 'error', 'schema validation failed:\n\t{0}'.format(e.message)) if args['schematron']: # schematron validate schema_path = lib_repo.get_oval_def_schema( OvalGenerator.oval_schema_version) output.message('info', 'performing schematron validation') try: lib_xml.schematron_validate(args['outfile'], schema_path) output.message('info', 'schematron validation successful') except lib_xml.SchematronValidationError as e: output.message( 'error', 'schematron validation failed:\n\t{0}'.format( '\n\t'.join(e.messages))) # add timinig seconds_elapsed = time.time() - start_time output.message('time', '{0}'.format(format_duration(seconds_elapsed))) # output results output.write_json()
def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser( description='Powers website OVAL definitions search.') output_options = parser.add_argument_group('output options') output_options.add_argument( '--page', nargs='?', default=1, type=int, help='page number to return (1 based, default: 1)') output_options.add_argument('--page_length', nargs='?', default=50, type=int, help='number of items per page (default: 50)') criteria_options = parser.add_argument_group( 'search criteria', 'Provide at least one of the following criteria to determine which definition(s) ' + 'will be included in the results set. Results will include the intersection of matches for each parameter ' + 'supplied. When multiple values are supplied for one paramater, the parameter will ' + 'match definitions that match any provided value.') criteria_options.add_argument('--definition_id', nargs='*', dest='oval_id', help='match OVAL definition id(s)') criteria_options.add_argument('--title', nargs='*', dest='title', metavar='PHRASE', help='match phrase(s) in definition titles') criteria_options.add_argument('--description', nargs='*', dest='description', metavar='PHRASE', help='match phrase(s) in definition titles') criteria_options.add_argument( '--class', nargs='*', dest='class', help='filter by class(es): {0}'.format(', '.join( lib_repo.supported_definition_classes))) criteria_options.add_argument( '--status', nargs='*', dest='status', help='filter by status(es): {0}'.format(', '.join( lib_repo.supported_definition_statuses))) criteria_options.add_argument('--family', nargs='*', dest='family', help='filter by family(ies)') criteria_options.add_argument('--platform', nargs='*', dest='platforms', metavar='PLATFORM', help='filter by platform(s)') criteria_options.add_argument('--product', nargs='*', dest='products', metavar='PRODUCT', help='filter by product(s)') criteria_options.add_argument('--contributor', nargs='*', dest='contributors', metavar='NAME', help='filter by contributor(s)') criteria_options.add_argument('--organization', nargs='*', dest='organizations', metavar='NAME', help='filter by organization(s)') criteria_options.add_argument( '--reference_id', nargs='*', dest='reference_ids', metavar='REFERENCE_ID', help='filter by reference ids, e.g. CVE-2015-3306') criteria_options.add_argument( '--max_schema_version', nargs="?", dest='max_schema_version', metavar='SCHEMA_VERSION', help='filter by maximum oval schema version, e.g. 5.10') criteria_options.add_argument( '--all_definitions', default=False, action="store_true", help= 'include all definitions in the repository (do not specify any other filters)' ) criteria_options.add_argument( '--from', nargs='?', default='', metavar='YYYYMMDD', help='include elements revised on or after this day (format: YYYYMMDD)' ) criteria_options.add_argument( '--to', nargs='?', default='', metavar='YYYYMMDD', help='include elements revised on or before this day (format: YYYYMMDD)' ) criteria_options.add_argument( '--revision_type', nargs='*', metavar='TYPE', help= 'optionally specify the type of revisons that the --to and --from date should match: status_change, created, submitted, modified, any (default:any)' ) args = vars(parser.parse_args()) # get output object output = WebOutput() # get index definitions_index = lib_search.ThreadSafeDefinitionsIndex(output.message) definitions_index.no_output = True # contruct query from args query = {} for field in definitions_index.get_fieldnames(): if field in args and args[field]: query[field] = args[field] # add schema_version filter, if specified if args['max_schema_version']: query['min_schema_version'] = '[0 TO {0}]'.format( definitions_index.version_to_int(args['max_schema_version'])) # add date range and contributor/org filters, if specified if args['from'] or args['to'] or args['contributors'] or args[ 'organizations']: # get revisions index revisions_index = lib_search.ThreadSafeRevisionsIndex(output.message) if args['from'] or args['to']: revision_date_query = { 'date': revisions_index.format_daterange(args['from'], args['to']) } if args['revision_type'] and 'any' not in args['revision_type']: revision_date_query['type'] = args['revision_type'] filtered_oval_ids = revisions_index.get_definition_ids( revision_date_query) if args['contributors']: contributor_filtered_ids = revisions_index.get_definition_ids( {'contributor': args['contributors']}) filtered_oval_ids = filtered_oval_ids & contributor_filtered_ids if 'filtered_oval_ids' in locals( ) else contributor_filtered_ids if args['organizations']: organization_filtered_ids = revisions_index.get_definition_ids( {'organization': args['organizations']}) filtered_oval_ids = filtered_oval_ids & organization_filtered_ids if 'filtered_oval_ids' in locals( ) else organization_filtered_ids # add to query if 'oval_id' in query and query['oval_id']: # if oval_id(s) specified in args, get intersection with filtered oval ids query['oval_id'] = set(query['oval_id']) & filtered_oval_ids else: query['oval_id'] = filtered_oval_ids if not query['oval_id']: output.message('info', 'No matching OVAL definitions found. Aborting.') output.write_json() sys.exit(0) # --all_definitions OR at least one definition selection option must be specified if args['all_definitions'] and query: output.message( 'error', "The '--all_definitions' filter cannot be combined with any other filters." ) output.write_json() sys.exit(0) elif not (args['all_definitions'] or query): output.message( 'error', 'At least one definitions filtering argument must be provided.') output.write_json() sys.exit(0) # query index query_results = definitions_index.paged_query(query, args['page'], args['page_length']) if not query_results or not query_results['documents']: output.message('info', 'No matching OVAL definitions found. Aborting.') output.write_json() sys.exit(0) # add results to output for query_result in query_results['documents']: del query_result['path'] for split_field in ['platforms', 'products', 'reference_ids']: query_result[split_field] = query_result[split_field].split( ',') if query_result[split_field] else [] query_result[ 'last_modified'] = query_result['last_modified'].isoformat( ) if query_result['last_modified'] else '' output.add_result(query_result) # add paging metadata output.add_keyvalue('page', query_results['page']) output.add_keyvalue('page_count', query_results['page_count']) output.add_keyvalue('page_length', query_results['page_length']) # add timinig seconds_elapsed = time.time() - start_time output.message('time', '{0}'.format(format_duration(seconds_elapsed))) # output results output.write_json()