def main():
""" parse command line options and generate statistics """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(
description='Get repository content statistics for website.')
groups_options = parser.add_argument_group(
'groups',
'Specify one or more groups to only show those groups. Displays all by default.'
)
for field in grouping_fields.keys():
label = grouping_fields[field]['label']
groups_options.add_argument('--{0}'.format(label),
default=False,
action="store_true",
help='show totals by {0}'.format(label))
args = vars(parser.parse_args())
# get list of groups specified in args
show_grouping_fields = [
group for group in grouping_fields.keys()
if args[grouping_fields[group]['label']]
]
# if not groups specified, show all
if not show_grouping_fields:
show_grouping_fields = grouping_fields.keys()
# get output object
output = WebOutput()
# get index references
index_refs = {
'definitions': lib_search.ThreadSafeDefinitionsIndex(output.message),
'revisions': lib_search.ThreadSafeRevisionsIndex(output.message),
'elements': lib_search.ThreadSafeElementsIndex(output.message)
}
for index in index_refs:
index_refs[index].no_output = True
# output the groups we're showing
total_groups = len(show_grouping_fields)
for i, group in enumerate(show_grouping_fields):
index = index_refs[grouping_fields[group]['index']]
group_totals = get_grouped_totals(group, index)
output.add_result(group, group_totals)
# add timinig
seconds_elapsed = time.time() - start_time
output.message('time', '{0}'.format(format_duration(seconds_elapsed)))
# output results
output.write_json()
def main():
""" parse command line options and generate statistics """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(
description='Get repository contributor statistics for website.')
filter_options = parser.add_argument_group('filter options')
filter_options.add_argument(
'--from',
nargs='?',
default='',
metavar='YYYYMMDD',
help='omit contributions before this day (format: YYYYMMDD)')
filter_options.add_argument(
'--to',
nargs='?',
default='',
metavar='YYYYMMDD',
help='omit contributions after this day (format: YYYYMMDD)')
args = vars(parser.parse_args())
# get output object
output = WebOutput()
# get index
revisions_index = lib_search.ThreadSafeRevisionsIndex(output.message)
revisions_index.no_output = True
# construct query
if args['from'] or args['to']:
date_range = "{0} TO {1}".format(args['from'] or '', args['to']
or '').strip()
query = {"date": '[{0}]'.format(date_range)}
else:
query = {}
# output the groups we're showing
results = revisions_index.grouped_query(query, ['contributor', 'type'])
output.add_result('contributor', format_grouped_totals(results))
results = revisions_index.grouped_query(query, ['organization', 'type'])
output.add_result('organization', format_grouped_totals(results))
# add timinig
seconds_elapsed = time.time() - start_time
output.message('time', '{0}'.format(format_duration(seconds_elapsed)))
# output results
output.write_json()
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(
description=
'Updates indexes. Should be run one each time new OVAL content is published.'
)
parser.add_argument('--nocache',
required=False,
action="store_true",
help='Rebuild from scratch even if not necessary')
args = vars(parser.parse_args())
# get output object
output = WebOutput()
# get definitions index and update
index = lib_search.ThreadSafeDefinitionsIndex(output.message)
index.no_output = True
index.update(args['nocache'])
output.message('definitions', 'index updated')
# get elements index and update
index = lib_search.ThreadSafeElementsIndex(output.message)
index.no_output = True
index.update(args['nocache'])
output.message('elements', 'index updated')
# get elements index and update
index = lib_search.ThreadSafeRevisionsIndex(output.message)
index.no_output = True
index.update(args['nocache'])
output.message('revisions', 'index updated')
# add timinig
seconds_elapsed = time.time() - start_time
output.message('time', '{0}'.format(format_duration(seconds_elapsed)))
# output results
output.write_json()
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(
description=
'Powers website-intiated builds if schema-valid OVAL definitions file. Note: unlike build_oval_definitions_file.py, this script will not automatically build/update the indices. You must run web_update_indexes.py before running this script to ensure you get accurate results.'
)
output_options = parser.add_argument_group('output options')
output_options.add_argument(
'-o',
'--outfile',
required=True,
help='file name for output OVAL definitions file')
output_options.add_argument('-v',
'--validate',
default=False,
action="store_true",
help='schema validate the output file')
output_options.add_argument('-s',
'--schematron',
default=False,
action="store_true",
help='schematron validate the output file')
output_options.add_argument(
'-t',
'--tempdir',
required=False,
default="./",
help=
"directory to store temporary files used when building oval definitions (default: './')"
)
source_options = parser.add_argument_group(
'definitions filtering',
'Provide at least one of the following options to determine which definition(s) '
+
'will be included. Results will include the intersection of matches for each parameter '
+
'supplied. When multiple values are supplied for one paramater, the parameter will '
+ 'match definitions that match any provided value.')
source_options.add_argument('--definition_id',
nargs='*',
dest='oval_id',
help='match OVAL definition id(s)')
source_options.add_argument('--title',
nargs='*',
dest='title',
metavar='PHRASE',
help='match phrase(s) in definition titles')
source_options.add_argument('--description',
nargs='*',
dest='description',
metavar='PHRASE',
help='match phrase(s) in definition titles')
source_options.add_argument(
'--class',
nargs='*',
dest='class',
help='filter by class(es): {0}'.format(', '.join(
lib_repo.supported_definition_classes)))
source_options.add_argument(
'--status',
nargs='*',
dest='status',
help='filter by status(es): {0}'.format(', '.join(
lib_repo.supported_definition_statuses)))
source_options.add_argument('--family',
nargs='*',
dest='family',
help='filter by family(ies)')
source_options.add_argument('--platform',
nargs='*',
dest='platforms',
metavar='PLATFORM',
help='filter by platform(s)')
source_options.add_argument('--product',
nargs='*',
dest='products',
metavar='PRODUCT',
help='filter by product(s)')
source_options.add_argument('--contributor',
nargs='*',
dest='contributors',
metavar='NAME',
help='filter by contributor(s)')
source_options.add_argument('--organization',
nargs='*',
dest='organizations',
metavar='NAME',
help='filter by organization(s)')
source_options.add_argument(
'--reference_id',
nargs='*',
dest='reference_ids',
metavar='REFERENCE_ID',
help='filter by reference ids, e.g. CVE-2015-3306')
source_options.add_argument(
'--max_schema_version',
nargs="?",
dest='max_schema_version',
metavar='SCHEMA_VERSION',
help='filter by maximum oval schema version, e.g. 5.10')
source_options.add_argument(
'--all_definitions',
default=False,
action="store_true",
help=
'include all definitions in the repository (do not specify any other filters)'
)
source_options.add_argument(
'--from',
nargs='?',
default='',
metavar='YYYYMMDD',
help='include elements revised on or after this day (format: YYYYMMDD)'
)
source_options.add_argument(
'--to',
nargs='?',
default='',
metavar='YYYYMMDD',
help='include elements revised on or before this day (format: YYYYMMDD)'
)
args = vars(parser.parse_args())
# get output object
output = WebOutput()
# get index
definitions_index = lib_search.ThreadSafeDefinitionsIndex(output.message)
definitions_index.no_output = True
# contruct query from args
query = {}
for field in definitions_index.get_fieldnames():
if field in args and args[field]:
query[field] = args[field]
# add schema_version filter, if specified
if args['max_schema_version']:
query['min_schema_version'] = '[0 TO {0}]'.format(
definitions_index.version_to_int(args['max_schema_version']))
# add date range and contributor/org filters, if specified
all_definitions_filtered = False
if args['from'] or args['to'] or args['contributors'] or args[
'organizations']:
# get revisions index
revisions_index = lib_search.ThreadSafeRevisionsIndex(output.message)
revisions_index.no_output = True
if args['from'] or args['to']:
filtered_oval_ids = revisions_index.get_definition_ids({
'date':
revisions_index.format_daterange(args['from'], args['to'])
})
if args['contributors']:
contributor_filtered_ids = revisions_index.get_definition_ids(
{'contributor': args['contributors']})
filtered_oval_ids = filtered_oval_ids & contributor_filtered_ids if 'filtered_oval_ids' in locals(
) else contributor_filtered_ids
if args['organizations']:
organization_filtered_ids = revisions_index.get_definition_ids(
{'organization': args['organizations']})
filtered_oval_ids = filtered_oval_ids & organization_filtered_ids if 'filtered_oval_ids' in locals(
) else organization_filtered_ids
# add to query
if 'oval_id' in query and query['oval_id']:
# if oval_id(s) specified in args, get intersection with filtered oval ids
query['oval_id'] = set(query['oval_id']) & filtered_oval_ids
else:
query['oval_id'] = filtered_oval_ids
if not query['oval_id']:
all_definitions_filtered = True
# --all_definitions OR at least one definition selection option must be specified
if args['all_definitions'] and query:
parser.print_help()
output.message(
'error',
"The '--all_definitions' filter cannot be combined with any other filters."
)
sys.exit(1)
elif not (args['all_definitions'] or query):
parser.print_help()
output.message(
'error',
'At least one definitions filtering argument must be provided.')
sys.exit(1)
# query index
query_results = definitions_index.query(
query) if not all_definitions_filtered else {}
# get set of all definition ids found
definition_ids = {document['oval_id'] for document in query_results}
output.message(
'info',
'Found {0} matching OVAL definitions'.format(len(definition_ids)))
# create generator and set oval schema version, if necessary
OvalGenerator = lib_xml.OvalGenerator(output.message, args['tempdir'])
if args['max_schema_version']:
OvalGenerator.oval_schema_version = args['max_schema_version']
if definition_ids:
# add all downstream element ids
output.message('info',
'Finding downstream OVAL ids for all definitions')
elements_index = lib_search.ThreadSafeElementsIndex(output.message)
elements_index.no_output = True
oval_ids = elements_index.find_downstream_ids(definition_ids,
definition_ids)
output.message(
'info', 'Found {0} downstream OVAL ids'.format(
len(oval_ids) - len(query_results)))
# get paths for all elements
output.message(
'info',
'Finding paths for {0} OVAL elements'.format(len(oval_ids)))
file_paths = elements_index.get_paths_from_ids(oval_ids)
# build in memory if there aren't that many files
if len(file_paths) < 200:
OvalGenerator.use_file_queues = False
# add each OVAL definition to generator
output.message(
'info', 'Generating OVAL definition file with {0} elements'.format(
len(oval_ids)))
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
# write output file
output.message('info',
'Writing OVAL definitions to {0}'.format(args['outfile']))
OvalGenerator.to_file(args['outfile'])
output.add_result(args['outfile'])
# validate
if args['validate']:
# schema validate
schema_path = lib_repo.get_oval_def_schema(
OvalGenerator.oval_schema_version)
output.message('info', 'performing schema validation')
try:
lib_xml.schema_validate(args['outfile'], schema_path)
output.message('info', 'schema validation successful')
except lib_xml.SchemaValidationError as e:
output.message(
'error', 'schema validation failed:\n\t{0}'.format(e.message))
if args['schematron']:
# schematron validate
schema_path = lib_repo.get_oval_def_schema(
OvalGenerator.oval_schema_version)
output.message('info', 'performing schematron validation')
try:
lib_xml.schematron_validate(args['outfile'], schema_path)
output.message('info', 'schematron validation successful')
except lib_xml.SchematronValidationError as e:
output.message(
'error', 'schematron validation failed:\n\t{0}'.format(
'\n\t'.join(e.messages)))
# add timinig
seconds_elapsed = time.time() - start_time
output.message('time', '{0}'.format(format_duration(seconds_elapsed)))
# output results
output.write_json()
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(
description='Powers website OVAL definitions search.')
output_options = parser.add_argument_group('output options')
output_options.add_argument(
'--page',
nargs='?',
default=1,
type=int,
help='page number to return (1 based, default: 1)')
output_options.add_argument('--page_length',
nargs='?',
default=50,
type=int,
help='number of items per page (default: 50)')
criteria_options = parser.add_argument_group(
'search criteria',
'Provide at least one of the following criteria to determine which definition(s) '
+
'will be included in the results set. Results will include the intersection of matches for each parameter '
+
'supplied. When multiple values are supplied for one paramater, the parameter will '
+ 'match definitions that match any provided value.')
criteria_options.add_argument('--definition_id',
nargs='*',
dest='oval_id',
help='match OVAL definition id(s)')
criteria_options.add_argument('--title',
nargs='*',
dest='title',
metavar='PHRASE',
help='match phrase(s) in definition titles')
criteria_options.add_argument('--description',
nargs='*',
dest='description',
metavar='PHRASE',
help='match phrase(s) in definition titles')
criteria_options.add_argument(
'--class',
nargs='*',
dest='class',
help='filter by class(es): {0}'.format(', '.join(
lib_repo.supported_definition_classes)))
criteria_options.add_argument(
'--status',
nargs='*',
dest='status',
help='filter by status(es): {0}'.format(', '.join(
lib_repo.supported_definition_statuses)))
criteria_options.add_argument('--family',
nargs='*',
dest='family',
help='filter by family(ies)')
criteria_options.add_argument('--platform',
nargs='*',
dest='platforms',
metavar='PLATFORM',
help='filter by platform(s)')
criteria_options.add_argument('--product',
nargs='*',
dest='products',
metavar='PRODUCT',
help='filter by product(s)')
criteria_options.add_argument('--contributor',
nargs='*',
dest='contributors',
metavar='NAME',
help='filter by contributor(s)')
criteria_options.add_argument('--organization',
nargs='*',
dest='organizations',
metavar='NAME',
help='filter by organization(s)')
criteria_options.add_argument(
'--reference_id',
nargs='*',
dest='reference_ids',
metavar='REFERENCE_ID',
help='filter by reference ids, e.g. CVE-2015-3306')
criteria_options.add_argument(
'--max_schema_version',
nargs="?",
dest='max_schema_version',
metavar='SCHEMA_VERSION',
help='filter by maximum oval schema version, e.g. 5.10')
criteria_options.add_argument(
'--all_definitions',
default=False,
action="store_true",
help=
'include all definitions in the repository (do not specify any other filters)'
)
criteria_options.add_argument(
'--from',
nargs='?',
default='',
metavar='YYYYMMDD',
help='include elements revised on or after this day (format: YYYYMMDD)'
)
criteria_options.add_argument(
'--to',
nargs='?',
default='',
metavar='YYYYMMDD',
help='include elements revised on or before this day (format: YYYYMMDD)'
)
criteria_options.add_argument(
'--revision_type',
nargs='*',
metavar='TYPE',
help=
'optionally specify the type of revisons that the --to and --from date should match: status_change, created, submitted, modified, any (default:any)'
)
args = vars(parser.parse_args())
# get output object
output = WebOutput()
# get index
definitions_index = lib_search.ThreadSafeDefinitionsIndex(output.message)
definitions_index.no_output = True
# contruct query from args
query = {}
for field in definitions_index.get_fieldnames():
if field in args and args[field]:
query[field] = args[field]
# add schema_version filter, if specified
if args['max_schema_version']:
query['min_schema_version'] = '[0 TO {0}]'.format(
definitions_index.version_to_int(args['max_schema_version']))
# add date range and contributor/org filters, if specified
if args['from'] or args['to'] or args['contributors'] or args[
'organizations']:
# get revisions index
revisions_index = lib_search.ThreadSafeRevisionsIndex(output.message)
if args['from'] or args['to']:
revision_date_query = {
'date':
revisions_index.format_daterange(args['from'], args['to'])
}
if args['revision_type'] and 'any' not in args['revision_type']:
revision_date_query['type'] = args['revision_type']
filtered_oval_ids = revisions_index.get_definition_ids(
revision_date_query)
if args['contributors']:
contributor_filtered_ids = revisions_index.get_definition_ids(
{'contributor': args['contributors']})
filtered_oval_ids = filtered_oval_ids & contributor_filtered_ids if 'filtered_oval_ids' in locals(
) else contributor_filtered_ids
if args['organizations']:
organization_filtered_ids = revisions_index.get_definition_ids(
{'organization': args['organizations']})
filtered_oval_ids = filtered_oval_ids & organization_filtered_ids if 'filtered_oval_ids' in locals(
) else organization_filtered_ids
# add to query
if 'oval_id' in query and query['oval_id']:
# if oval_id(s) specified in args, get intersection with filtered oval ids
query['oval_id'] = set(query['oval_id']) & filtered_oval_ids
else:
query['oval_id'] = filtered_oval_ids
if not query['oval_id']:
output.message('info',
'No matching OVAL definitions found. Aborting.')
output.write_json()
sys.exit(0)
# --all_definitions OR at least one definition selection option must be specified
if args['all_definitions'] and query:
output.message(
'error',
"The '--all_definitions' filter cannot be combined with any other filters."
)
output.write_json()
sys.exit(0)
elif not (args['all_definitions'] or query):
output.message(
'error',
'At least one definitions filtering argument must be provided.')
output.write_json()
sys.exit(0)
# query index
query_results = definitions_index.paged_query(query, args['page'],
args['page_length'])
if not query_results or not query_results['documents']:
output.message('info', 'No matching OVAL definitions found. Aborting.')
output.write_json()
sys.exit(0)
# add results to output
for query_result in query_results['documents']:
del query_result['path']
for split_field in ['platforms', 'products', 'reference_ids']:
query_result[split_field] = query_result[split_field].split(
',') if query_result[split_field] else []
query_result[
'last_modified'] = query_result['last_modified'].isoformat(
) if query_result['last_modified'] else ''
output.add_result(query_result)
# add paging metadata
output.add_keyvalue('page', query_results['page'])
output.add_keyvalue('page_count', query_results['page_count'])
output.add_keyvalue('page_length', query_results['page_length'])
# add timinig
seconds_elapsed = time.time() - start_time
output.message('time', '{0}'.format(format_duration(seconds_elapsed)))
# output results
output.write_json()