def main():
today = datetime.date.today()
url = r"http://oval.mitre.org/rep-data/5.10/org.mitre.oval/oval.xml"
dest = r"C:\_Development\oval_repository\Serializer\processed\oval-%s-%s-%s.xml" % (
today.year, today.month, today.day)
(fn, msg) = urllib.request.urlretrieve(url, dest)
message("INFO", "Downloaded file %s" % fn)
oval_decomposition.decompose(fn, True)
# get indexes
definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
for defpath in lib_repo.get_definition_paths_iterator():
message("INFO",
"Determining minimum schema version for file %s" % defpath)
result = lib_schema.determine_def_min_version(defpath,
definitions_index,
elements_index, True)
message(
"INFO",
"The minimum OVAL version supported by '{0}' is {1}.".format(
defpath, result['minimum_version']))
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(description='Determine minimum supported OVAL schema for a given definition')
parser.add_argument('-p', '--path', help='path to definition fragment')
parser.add_argument('-i', '--id', help='id of definition fragment')
parser.add_argument('-a', '--all', default=False, action="store_true", help='determine minimum supported OVAL schema for all indexed definition fragments. cannot be combined with --id or --path')
parser.add_argument('-u', '--update', default=False, action="store_true", help='update the given definition to include the min_schema_version element')
args = vars(parser.parse_args())
if (args['all'] and (args['id'] or args['path'])):
parser.print_help()
message('error', "--all cannot be combined with --id or --path.")
sys.exit(0)
if not args['all'] and not (args['path'] or args['id']):
parser.print_help()
message('error',"Please specify one of: --all, --path or --id.")
sys.exit(0)
if (args['path'] and args['id']):
parser.print_help()
message('error',"Please specify one of: --all, --path, or --id.")
sys.exit(0)
# get indexes
definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
if args['all']:
for defpath in lib_repo.get_definition_paths_iterator():
result = determine_def_min_version(defpath, definitions_index, elements_index, args['update'])
report(defpath, result['minimum_version'])
elif args['id']:
documents = definitions_index.query({ 'oval_id': args['id'] })
if len(documents) == 0:
message('info',"No definitions having id '{0}' found.".format(args['id']))
sys.exit(0)
paths = [ document['path'] for document in documents ]
for path in paths:
result = determine_def_min_version(path, definitions_index, elements_index, args['update'])
report(result['oval_id'], result['minimum_version'])
elif args['path'] and not os.path.isfile(args['path']):
message('info',"Definition fragment path not found: " + args['path'])
sys.exit(0)
else:
result = determine_def_min_version(args['path'], definitions_index, elements_index, args['update'])
report(args['path'], result['minimum_version'])
seconds_elapsed = time.time() - start_time
report('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(
description='Builds all OVAL definitons in the repository.')
parser.add_argument(
'-l',
'--limit',
nargs='?',
default="0",
type=int,
help='limits number of definitions that will be built)')
args = vars(parser.parse_args())
# get indexes
definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
# create generator, build in memory b/c smallish files
OvalGenerator = lib_xml.OvalGenerator(message)
OvalGenerator.use_file_queues = False
i_file = 0
i_limit = args['limit']
for defpath in lib_repo.get_definition_paths_iterator():
i_file = i_file + 1
if i_limit > 0 and i_file > i_limit:
break
def_id = lib_repo.path_to_oval_id(defpath)
message('info', 'Building file {0} for {1}.'.format(i_file, def_id))
# add all downstream element ids
def_ids = set([def_id])
oval_ids = elements_index.find_downstream_ids(def_ids, def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
# add each OVAL definition to generator
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
# write output file
outfile = './tmp/gen.{0}'.format(lib_repo.oval_id_to_path(def_id))
OvalGenerator.to_file(outfile)
seconds_elapsed = time.time() - start_time
message('info',
'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def get_definition_path(self):
oval_query = {'reference_ids': self.cve_id}
query_results = lib_search.DefinitionsIndex().query(oval_query)
if not query_results:
return False
# TODO support other classes
for item in query_results:
if item['class'] == 'vulnerability':
self.xml_path = item['path']
else:
continue
return True
def main():
query = {}
definitions_index = lib_search.DefinitionsIndex(message)
#revisions_index = lib_search.RevisionsIndex(message)
#filtered_oval_ids = revisions_index.get_definition_ids({ 'date': revisions_index.format_daterange("20151201", "20151231") })
#filtered_oval_ids = revisions_index.get_definition_ids({ 'status': 'INITIAL SUBMISSION' })
#for ovalid in filtered_oval_ids:
# message("INFO", "Filtered oval id is %s" % ovalid)
query["status"] = "INITIAL SUBMISSION"
#query['oval_id'] = filtered_oval_ids
query_results = definitions_index.query(query)
#definition_ids = { document['oval_id'] for document in query_results }
for document in query_results:
message("INFO", "OVAL ID %s has status %s" % (document["oval_id"], document["status"]))
def main():
start_time = time.time()
definitions_index = lib_search.DefinitionsIndex(message)
ns_map = {'oval-def': 'http://oval.mitre.org/XMLSchema/oval-definitions-5'}
# Create the output object...
output = WebOutput()
for defpath in lib_repo.get_definition_paths_iterator():
def_id = lib_repo.path_to_oval_id(defpath)
try:
tree = etree.parse(defpath)
except OSError as e:
raise InvalidPathError(defpath)
except etree.XMLSyntaxError as e:
raise InvalidXmlError(defpath, e.msg)
root = tree.getroot()
# Only Vulnerability and Patch definitions would have CVE references...
defclass = root.get('class')
if defclass in ["vulnerability", "patch"]:
reference = root.find(
"./oval-def:metadata/oval-def:reference[@source='CVE']",
ns_map)
if reference is not None:
output.add_result(
def_id,
[reference.get("ref_id"),
reference.get("ref_url")])
# add timing
seconds_elapsed = time.time() - start_time
output.message("CVE", "OVAL Repository Definition-to-CVE Mappings")
output.message("time", "{0}".format(format_duration(seconds_elapsed)))
# Write JSON Output...
output.write_json()
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(
description='Builds a schema-valid OVAL definitions file.')
output_options = parser.add_argument_group('output options')
output_options.add_argument(
'-o',
'--outfile',
required=True,
help='file name for output OVAL definitions file')
output_options.add_argument('-v',
'--validate',
default=False,
action="store_true",
help='schema validate the output file')
output_options.add_argument('-s',
'--schematron',
default=False,
action="store_true",
help='schematron validate the output file')
output_options.add_argument(
'-t',
'--tempdir',
required=False,
default="./",
help=
"directory to store temporary files used when building oval definitions (default: './')"
)
source_options = parser.add_argument_group(
'definitions filtering',
'Provide at least one of the following options to determine which definition(s) '
+
'will be included. Results will include the intersection of matches for each parameter '
+
'supplied. When multiple values are supplied for one paramater, the parameter will '
+ 'match definitions that match any provided value.')
source_options.add_argument('--definition_id',
nargs='*',
dest='oval_id',
help='match OVAL definition id(s)')
source_options.add_argument('--title',
nargs='*',
dest='title',
metavar='PHRASE',
help='match phrase(s) in definition titles')
source_options.add_argument('--description',
nargs='*',
dest='description',
metavar='PHRASE',
help='match phrase(s) in definition titles')
source_options.add_argument(
'--class',
nargs='*',
dest='class',
help='filter by class(es): {0}'.format(', '.join(
lib_repo.supported_definition_classes)))
source_options.add_argument(
'--status',
nargs='*',
dest='status',
help='filter by status(es): {0}'.format(', '.join(
lib_repo.supported_definition_statuses)))
source_options.add_argument('--family',
nargs='*',
dest='family',
help='filter by family(ies)')
source_options.add_argument('--platform',
nargs='*',
dest='platforms',
metavar='PLATFORM',
help='filter by platform(s)')
source_options.add_argument('--product',
nargs='*',
dest='products',
metavar='PRODUCT',
help='filter by product(s)')
source_options.add_argument('--contributor',
nargs='*',
dest='contributors',
metavar='NAME',
help='filter by contributor(s)')
source_options.add_argument('--organization',
nargs='*',
dest='organizations',
metavar='NAME',
help='filter by organization(s)')
source_options.add_argument(
'--reference_id',
nargs='*',
dest='reference_ids',
metavar='REFERENCE_ID',
help='filter by reference ids, e.g. CVE-2015-3306')
source_options.add_argument(
'--max_schema_version',
nargs="?",
dest='max_schema_version',
metavar='SCHEMA_VERSION',
help='filter by maximum oval schema version, e.g. 5.10')
source_options.add_argument(
'--all_definitions',
default=False,
action="store_true",
help=
'include all definitions in the repository (do not specify any other filters)'
)
source_options.add_argument(
'--from',
nargs='?',
default='',
metavar='YYYYMMDD',
help='include elements revised on or after this day (format: YYYYMMDD)'
)
source_options.add_argument(
'--to',
nargs='?',
default='',
metavar='YYYYMMDD',
help='include elements revised on or before this day (format: YYYYMMDD)'
)
args = vars(parser.parse_args())
# get definitions index
definitions_index = lib_search.DefinitionsIndex(message)
# contruct query from args
query = {}
for field in definitions_index.get_fieldnames():
if field in args and args[field]:
query[field] = args[field]
# add schema_version filter, if specified
if args['max_schema_version']:
query['min_schema_version'] = '[0 TO {0}]'.format(
definitions_index.version_to_int(args['max_schema_version']))
# add date range and contributor/org filters, if specified
all_definitions_filtered = False
if args['from'] or args['to'] or args['contributors'] or args[
'organizations']:
# get revisions index
revisions_index = lib_search.RevisionsIndex(message)
if args['from'] or args['to']:
filtered_oval_ids = revisions_index.get_definition_ids({
'date':
revisions_index.format_daterange(args['from'], args['to'])
})
if args['contributors']:
contributor_filtered_ids = revisions_index.get_definition_ids(
{'contributor': args['contributors']})
filtered_oval_ids = filtered_oval_ids & contributor_filtered_ids if 'filtered_oval_ids' in locals(
) else contributor_filtered_ids
if args['organizations']:
organization_filtered_ids = revisions_index.get_definition_ids(
{'organization': args['organizations']})
filtered_oval_ids = filtered_oval_ids & organization_filtered_ids if 'filtered_oval_ids' in locals(
) else organization_filtered_ids
# add to query
if 'oval_id' in query and query['oval_id']:
# if oval_id(s) specified in args, get intersection with filtered oval ids
query['oval_id'] = set(query['oval_id']) & filtered_oval_ids
else:
query['oval_id'] = filtered_oval_ids
if not query['oval_id']:
all_definitions_filtered = True
# --all_definitions OR at least one definition selection option must be specified
if args['all_definitions'] and query:
parser.print_help()
message(
'error',
"The '--all_definitions' filter cannot be combined with any other filters."
)
sys.exit(1)
elif not (args['all_definitions'] or query):
parser.print_help()
message(
'error',
'At least one definitions filtering argument must be provided.')
sys.exit(1)
# query index
query_results = definitions_index.query(
query) if not all_definitions_filtered else {}
# get set of all definition ids found
definition_ids = {document['oval_id'] for document in query_results}
message('info',
'Found {0} matching OVAL definitions'.format(len(definition_ids)))
# create generator and set oval schema version, if necessary
OvalGenerator = lib_xml.OvalGenerator(message, args['tempdir'])
if args['max_schema_version']:
OvalGenerator.oval_schema_version = args['max_schema_version']
if definition_ids:
# add all downstream element ids
message('info', 'Finding downstream OVAL ids for all definitions')
elements_index = lib_search.ElementsIndex(message)
oval_ids = elements_index.find_downstream_ids(definition_ids,
definition_ids)
message(
'info', 'Found {0} downstream OVAL ids'.format(
len(oval_ids) - len(query_results)))
# get paths for all elements
message('info',
'Finding paths for {0} OVAL elements'.format(len(oval_ids)))
file_paths = elements_index.get_paths_from_ids(oval_ids)
# build in memory if there aren't that many files
if len(file_paths) < 200:
OvalGenerator.use_file_queues = False
# add each OVAL definition to generator
message(
'info', 'Generating OVAL definition file with {0} elements'.format(
len(oval_ids)))
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
# write output file
message('info', 'Writing OVAL definitions to {0}'.format(args['outfile']))
OvalGenerator.to_file(args['outfile'])
# validate
if args['validate']:
# schema validate
schema_path = lib_repo.get_oval_def_schema(
OvalGenerator.oval_schema_version)
message('info', 'performing schema validation')
try:
lib_xml.schema_validate(args['outfile'], schema_path)
message('info', 'schema validation successful')
except lib_xml.SchemaValidationError as e:
message('error',
'schema validation failed:\n\t{0}'.format(e.message))
if args['schematron']:
# schematron validate
schema_path = lib_repo.get_oval_def_schema(
OvalGenerator.oval_schema_version)
message('info', 'performing schematron validation')
try:
lib_xml.schematron_validate(args['outfile'], schema_path)
message('info', 'schematron validation successful')
except lib_xml.SchematronValidationError as e:
message(
'error', 'schematron validation failed:\n\t{0}'.format(
'\n\t'.join(e.messages)))
seconds_elapsed = time.time() - start_time
message('info',
'Completed in {0}!'.format(format_duration(seconds_elapsed)))