def main():
today = datetime.date.today()
url = r"http://oval.mitre.org/rep-data/5.10/org.mitre.oval/oval.xml"
dest = r"C:\_Development\oval_repository\Serializer\processed\oval-%s-%s-%s.xml" % (
today.year, today.month, today.day)
(fn, msg) = urllib.request.urlretrieve(url, dest)
message("INFO", "Downloaded file %s" % fn)
oval_decomposition.decompose(fn, True)
# get indexes
definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
for defpath in lib_repo.get_definition_paths_iterator():
message("INFO",
"Determining minimum schema version for file %s" % defpath)
result = lib_schema.determine_def_min_version(defpath,
definitions_index,
elements_index, True)
message(
"INFO",
"The minimum OVAL version supported by '{0}' is {1}.".format(
defpath, result['minimum_version']))
def main():
start_time = time.time()
definitions_index = lib_search.DefinitionsIndex(message)
ns_map = {"oval-def": "http://oval.mitre.org/XMLSchema/oval-definitions-5"}
# Create the output object...
output = WebOutput()
for defpath in lib_repo.get_definition_paths_iterator():
def_id = lib_repo.path_to_oval_id(defpath)
try:
tree = etree.parse(defpath)
except OSError as e:
raise InvalidPathError(defpath)
except etree.XMLSyntaxError as e:
raise InvalidXmlError(defpath, e.msg)
root = tree.getroot()
# Only Vulnerability and Patch definitions would have CVE references...
defclass = root.get("class")
if defclass in ["vulnerability", "patch"]:
reference = root.find("./oval-def:metadata/oval-def:reference[@source='CVE']", ns_map)
if reference is not None:
output.add_result(def_id, [reference.get("ref_id"), reference.get("ref_url")])
# add timing
seconds_elapsed = time.time() - start_time
output.message("CVE", "OVAL Repository Definition-to-CVE Mappings")
output.message("time", "{0}".format(format_duration(seconds_elapsed)))
# Write JSON Output...
output.write_json()
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(description='Determine minimum supported OVAL schema for a given definition')
parser.add_argument('-p', '--path', help='path to definition fragment')
parser.add_argument('-i', '--id', help='id of definition fragment')
parser.add_argument('-a', '--all', default=False, action="store_true", help='determine minimum supported OVAL schema for all indexed definition fragments. cannot be combined with --id or --path')
parser.add_argument('-u', '--update', default=False, action="store_true", help='update the given definition to include the min_schema_version element')
args = vars(parser.parse_args())
if (args['all'] and (args['id'] or args['path'])):
parser.print_help()
message('error', "--all cannot be combined with --id or --path.")
sys.exit(0)
if not args['all'] and not (args['path'] or args['id']):
parser.print_help()
message('error',"Please specify one of: --all, --path or --id.")
sys.exit(0)
if (args['path'] and args['id']):
parser.print_help()
message('error',"Please specify one of: --all, --path, or --id.")
sys.exit(0)
# get indexes
definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
if args['all']:
for defpath in lib_repo.get_definition_paths_iterator():
result = determine_def_min_version(defpath, definitions_index, elements_index, args['update'])
report(defpath, result['minimum_version'])
elif args['id']:
documents = definitions_index.query({ 'oval_id': args['id'] })
if len(documents) == 0:
message('info',"No definitions having id '{0}' found.".format(args['id']))
sys.exit(0)
paths = [ document['path'] for document in documents ]
for path in paths:
result = determine_def_min_version(path, definitions_index, elements_index, args['update'])
report(result['oval_id'], result['minimum_version'])
elif args['path'] and not os.path.isfile(args['path']):
message('info',"Definition fragment path not found: " + args['path'])
sys.exit(0)
else:
result = determine_def_min_version(args['path'], definitions_index, elements_index, args['update'])
report(args['path'], result['minimum_version'])
seconds_elapsed = time.time() - start_time
report('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(description='Determine minimum supported OVAL schema for a given definition')
parser.add_argument('-p', '--path', help='path to definition fragment')
parser.add_argument('-i', '--id', help='id of definition fragment')
parser.add_argument('-a', '--all', default=False, action="store_true", help='determine minimum supported OVAL schema for all indexed definition fragments. cannot be combined with --id or --path')
parser.add_argument('-u', '--update', default=False, action="store_true", help='update the given definition to include the min_schema_version element')
args = vars(parser.parse_args())
if (args['all'] and (args['id'] or args['path'])):
parser.print_help()
message('error', "--all cannot be combined with --id or --path.")
sys.exit(0)
if not args['all'] and not (args['path'] or args['id']):
parser.print_help()
message('error',"Please specify one of: --all, --path or --id.")
sys.exit(0)
if (args['path'] and args['id']):
parser.print_help()
message('error',"Please specify one of: --all, --path, or --id.")
sys.exit(0)
# get indexes
definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
if args['all']:
for defpath in lib_repo.get_definition_paths_iterator():
result = determine_def_min_version(defpath, definitions_index, elements_index, args['update'])
report(defpath, result['minimum_version'])
elif args['id']:
documents = definitions_index.query({ 'oval_id': args['id'] })
if len(documents) == 0:
message('info',"No definitions having id '{0}' found.".format(args['id']))
sys.exit(0)
paths = [ document['path'] for document in documents ]
for path in paths:
result = determine_def_min_version(path, definitions_index, elements_index, args['update'])
report(result['oval_id'], result['minimum_version'])
elif args['path'] and not os.path.isfile(args['path']):
message('info',"Definition fragment path not found: " + args['path'])
sys.exit(0)
else:
result = determine_def_min_version(args['path'], definitions_index, elements_index, args['update'])
report(args['path'], result['minimum_version'])
seconds_elapsed = time.time() - start_time
report('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(
description='Builds all OVAL definitons in the repository.')
parser.add_argument(
'-l',
'--limit',
nargs='?',
default="0",
type=int,
help='limits number of definitions that will be built)')
args = vars(parser.parse_args())
# get indexes
# definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
# create generator, build in memory b/c smallish files
OvalGenerator = lib_xml.OvalGenerator(message)
OvalGenerator.use_file_queues = False
print("Storing data in temp directory: {}".format(tempfile.gettempdir()))
i_file = 0
i_limit = args['limit']
for defpath in lib_repo.get_definition_paths_iterator():
i_file = i_file + 1
if i_limit > 0 and i_file > i_limit:
break
def_id = lib_repo.path_to_oval_id(defpath)
message('info', 'Building file {0} for {1}.'.format(i_file, def_id))
# add all downstream element ids
def_ids = set([def_id])
oval_ids = elements_index.find_downstream_ids(def_ids, def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
# add each OVAL definition to generator
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
# write output file
outfile = '{1}/gen.{0}'.format(lib_repo.oval_id_to_path(def_id),
tempfile.gettempdir())
OvalGenerator.to_file(outfile)
seconds_elapsed = time.time() - start_time
message('info',
'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def document_iterator(self, paths=False):
""" Iterator yielding definition files in repo as indexable documents. """
if not paths:
# get all in repo
paths = lib_repo.get_definition_paths_iterator()
for path in paths:
try:
document = lib_xml.get_definition_metadata(path)
except lib_xml.InvalidPathError as e:
yield { 'path': e.path, 'deleted': True }
else:
document = self.whoosh_escape_document(document)
yield document
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(description="Builds all OVAL definitons in the repository.")
parser.add_argument(
"-l", "--limit", nargs="?", default="0", type=int, help="limits number of definitions that will be built)"
)
args = vars(parser.parse_args())
# get indexes
definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
# create generator, build in memory b/c smallish files
OvalGenerator = lib_xml.OvalGenerator(message)
OvalGenerator.use_file_queues = False
i_file = 0
i_limit = args["limit"]
for defpath in lib_repo.get_definition_paths_iterator():
i_file = i_file + 1
if i_limit > 0 and i_file > i_limit:
break
def_id = lib_repo.path_to_oval_id(defpath)
message("info", "Building file {0} for {1}.".format(i_file, def_id))
# add all downstream element ids
def_ids = set([def_id])
oval_ids = elements_index.find_downstream_ids(def_ids, def_ids)
file_paths = elements_index.get_paths_from_ids(oval_ids)
# add each OVAL definition to generator
for file_path in file_paths:
element_type = lib_repo.get_element_type_from_path(file_path)
OvalGenerator.queue_element_file(element_type, file_path)
# write output file
outfile = "./tmp/gen.{0}".format(lib_repo.oval_id_to_path(def_id))
OvalGenerator.to_file(outfile)
seconds_elapsed = time.time() - start_time
message("info", "Completed in {0}!".format(format_duration(seconds_elapsed)))
def document_iterator(self, paths=False):
""" Iterator yielding definition files in repo as indexable documents. """
if not paths:
# get all in repo
paths = lib_repo.get_definition_paths_iterator()
for path in paths:
try:
document = lib_xml.get_definition_metadata(path)
except lib_xml.InvalidPathError as e:
yield { 'path': e.path, 'deleted': True }
except lib_xml.InvalidXmlError as e:
raise
else:
del document['revisions']
document = self.whoosh_escape_document(document)
document['min_schema_version'] = self.version_to_int(document['min_schema_version'])
yield document
def document_iterator(self, paths=False):
""" Iterator yielding definition files in repo as indexable documents. """
if not paths:
# get all in repo
paths = lib_repo.get_definition_paths_iterator()
for path in paths:
try:
document = lib_xml.get_definition_metadata(path)
except lib_xml.InvalidPathError as e:
yield { 'path': e.path, 'deleted': True }
except lib_xml.InvalidXmlError as e:
raise
else:
del document['revisions']
document = self.whoosh_escape_document(document)
document['min_schema_version'] = self.version_to_int(document['min_schema_version'])
yield document
def document_iterator(self, paths=False):
""" Iterator yielding definition revisions in repo as indexable documents. """
if not paths:
# get all in repo
paths = lib_repo.get_definition_paths_iterator()
for path in paths:
try:
document = lib_xml.get_definition_metadata(path)
except lib_xml.InvalidPathError as e:
yield { 'oval_id': lib_repo.path_to_oval_id(e.path), 'deleted': True }
except lib_xml.InvalidXmlError as e:
raise
else:
oval_id = document['oval_id']
for revision in document['revisions']:
revision['oval_id'] = oval_id
revision = self.whoosh_escape_document(revision)
yield revision
def main():
today = datetime.date.today()
url = r"http://oval.mitre.org/rep-data/5.10/org.mitre.oval/oval.xml"
dest = r"C:\_Development\oval_repository\Serializer\processed\oval-%s-%s-%s.xml" % (today.year, today.month, today.day)
(fn, msg) = urllib.request.urlretrieve(url, dest)
message("INFO", "Downloaded file %s" % fn)
oval_decomposition.decompose(fn, True)
# get indexes
definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
for defpath in lib_repo.get_definition_paths_iterator():
message("INFO", "Determining minimum schema version for file %s" % defpath)
result = lib_schema.determine_def_min_version(defpath, definitions_index, elements_index, True)
message("INFO", "The minimum OVAL version supported by '{0}' is {1}.".format(defpath, result['minimum_version']))
def document_iterator(self, paths=False):
""" Iterator yielding definition revisions in repo as indexable documents. """
if not paths:
# get all in repo
paths = lib_repo.get_definition_paths_iterator()
for path in paths:
try:
document = lib_xml.get_definition_metadata(path)
except lib_xml.InvalidPathError as e:
yield { 'oval_id': lib_repo.path_to_oval_id(e.path), 'deleted': True }
except lib_xml.InvalidXmlError as e:
raise
else:
oval_id = document['oval_id']
for revision in document['revisions']:
revision['oval_id'] = oval_id
revision = self.whoosh_escape_document(revision)
yield revision
def main():
start_time = time.time()
definitions_index = lib_search.DefinitionsIndex(message)
ns_map = {'oval-def': 'http://oval.mitre.org/XMLSchema/oval-definitions-5'}
# Create the output object...
output = WebOutput()
for defpath in lib_repo.get_definition_paths_iterator():
def_id = lib_repo.path_to_oval_id(defpath)
try:
tree = etree.parse(defpath)
except OSError as e:
raise InvalidPathError(defpath)
except etree.XMLSyntaxError as e:
raise InvalidXmlError(defpath, e.msg)
root = tree.getroot()
# Only Vulnerability and Patch definitions would have CVE references...
defclass = root.get('class')
if defclass in ["vulnerability", "patch"]:
reference = root.find(
"./oval-def:metadata/oval-def:reference[@source='CVE']",
ns_map)
if reference is not None:
output.add_result(
def_id,
[reference.get("ref_id"),
reference.get("ref_url")])
# add timing
seconds_elapsed = time.time() - start_time
output.message("CVE", "OVAL Repository Definition-to-CVE Mappings")
output.message("time", "{0}".format(format_duration(seconds_elapsed)))
# Write JSON Output...
output.write_json()
def main():
start_time = time.time()
tracking = dict()
elements_index = lib_search.ElementsIndex(message)
# getting all definitions
all_def_ids = set()
message('info', 'getting all definitions')
for defpath in lib_repo.get_definition_paths_iterator():
all_def_ids.add(lib_repo.path_to_oval_id(defpath))
message('info', 'found {0} definitions'.format(len(all_def_ids)))
# getting all element ids downstream from any definition
message('info', 'getting all downstream element ids')
all_downstream_ids = elements_index.find_downstream_ids(all_def_ids)
message('info',
'found {0} downstream element ids'.format(len(all_downstream_ids)))
# get elements that aren't in all_downstream_ids
message('info', 'checking all elements')
cur_element_type = None
for elempath in lib_repo.get_element_paths_iterator():
oval_id = lib_repo.path_to_oval_id(elempath)
element_type = lib_repo.get_element_type_from_oval_id(oval_id)
# skip definitions... we're only pruning child elements
if element_type == 'definition':
continue
# write status msg
if element_type != cur_element_type:
cur_element_type = element_type
i_element_type = 0
i_element_type = i_element_type + 1
sys.stdout.write(
'Analyzing {0}s: {1} \r'.format(
cur_element_type, i_element_type))
sys.stdout.flush()
# it's an orphan if it's not downsteam of a definition
track_as = 'orphan' if oval_id not in all_downstream_ids else 'in_use'
if not track_as in tracking:
tracking[track_as] = dict()
if not element_type in tracking[track_as]:
tracking[track_as][element_type] = set()
tracking[track_as][element_type].add(oval_id)
sys.stdout.write(
' \r'.
format(cur_element_type, i_element_type))
sys.stdout.flush()
# generate report
report = []
for track_as, elements_by_type in tracking.items():
report.append('\t{0}:'.format(track_as.replace('_', ' ').capitalize()))
for element_type, oval_ids in elements_by_type.items():
report.append('\t\t{0} {1}s'.format(len(oval_ids), element_type))
message('found', '\n'.join(report))
response = input("\n :::: Remove all orphans? (N[o] / y[es]): ")
if response.lower() == 'y':
orphan_ids = set()
for element_type, oval_ids in tracking['orphan'].items():
orphan_ids.update(oval_ids)
file_paths = elements_index.get_paths_from_ids(orphan_ids)
for file_path in file_paths:
message("INFO",
"Deleting Orphan '%s'" % os.path.basename(file_path))
os.remove(file_path)
seconds_elapsed = time.time() - start_time
message('info',
'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main():
start_time = time.time()
tracking = dict()
elements_index = lib_search.ElementsIndex(message)
# getting all definitions
all_def_ids = set()
message('info', 'getting all definitions')
for defpath in lib_repo.get_definition_paths_iterator():
all_def_ids.add(lib_repo.path_to_oval_id(defpath))
message('info', 'found {0} definitions'.format(len(all_def_ids)))
# getting all element ids downstream from any definition
message('info', 'getting all downstream element ids')
all_downstream_ids = elements_index.find_downstream_ids(all_def_ids)
message('info', 'found {0} downstream element ids'.format(len(all_downstream_ids)))
# get elements that aren't in all_downstream_ids
message('info', 'checking all elements')
cur_element_type = None
for elempath in lib_repo.get_element_paths_iterator():
oval_id = lib_repo.path_to_oval_id(elempath)
element_type = lib_repo.get_element_type_from_oval_id(oval_id)
# skip definitions... we're only pruning child elements
if element_type == 'definition':
continue
# write status msg
if element_type != cur_element_type:
cur_element_type = element_type
i_element_type = 0
i_element_type = i_element_type + 1
sys.stdout.write('Analyzing {0}s: {1} \r'.format(cur_element_type, i_element_type))
sys.stdout.flush()
# it's an orphan if it's not downsteam of a definition
track_as = 'orphan' if oval_id not in all_downstream_ids else 'in_use'
if not track_as in tracking:
tracking[track_as] = dict()
if not element_type in tracking[track_as]:
tracking[track_as][element_type] = set()
tracking[track_as][element_type].add(oval_id)
sys.stdout.write(' \r'.format(cur_element_type, i_element_type))
sys.stdout.flush()
# generate report
report = []
for track_as, elements_by_type in tracking.items():
report.append('\t{0}:'.format(track_as.replace('_', ' ').capitalize()))
for element_type, oval_ids in elements_by_type.items():
report.append('\t\t{0} {1}s'.format(len(oval_ids), element_type))
message('found', '\n'.join(report))
response = input("\n :::: Remove all orphans? (N[o] / y[es]): ")
if response.lower() == 'y':
orphan_ids = set()
for element_type, oval_ids in tracking['orphan'].items():
orphan_ids.update(oval_ids)
file_paths = elements_index.get_paths_from_ids(orphan_ids)
for file_path in file_paths:
message("INFO", "Deleting Orphan '%s'" % os.path.basename(file_path))
os.remove(file_path)
seconds_elapsed = time.time() - start_time
message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main():
""" parse command line options and generate file """
start_time = time.time()
# parse command line options
parser = argparse.ArgumentParser(description="Determine minimum supported OVAL schema for a given definition")
parser.add_argument("-p", "--path", help="path to definition fragment")
parser.add_argument("-i", "--id", help="id of definition fragment")
parser.add_argument(
"-a",
"--all",
default=False,
action="store_true",
help="determine minimum supported OVAL schema for all indexed definitino fragments. cannot be combined with --id or --path",
)
parser.add_argument(
"-u",
"--update",
default=False,
action="store_true",
help="update the given definition to include the min_schema_version element",
)
args = vars(parser.parse_args())
if args["all"] and (args["id"] or args["path"]):
parser.print_help()
message("error", "--all cannot be combined with --id or --path.")
sys.exit(0)
if not args["all"] and not (args["path"] or args["id"]):
parser.print_help()
message("error", "Please specify one of: --all, --path or --id.")
sys.exit(0)
if args["path"] and args["id"]:
parser.print_help()
message("error", "Please specify one of: --all, --path, or --id.")
sys.exit(0)
# get indexes
definitions_index = lib_search.DefinitionsIndex(message)
elements_index = lib_search.ElementsIndex(message)
# build schema path cache
for schema_version in lib_repo.get_schema_versions():
schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)
if args["all"]:
for defpath in lib_repo.get_definition_paths_iterator():
result = determine_def_min_version(defpath, definitions_index, elements_index, args["update"])
report(defpath, result["minimum_version"])
elif args["id"]:
documents = definitions_index.query({"oval_id": args["id"]})
if len(documents) == 0:
message("info", "No definitions having id '{0}' found.".format(args["id"]))
sys.exit(0)
paths = [document["path"] for document in documents]
for path in paths:
result = determine_def_min_version(path, definitions_index, elements_index, args["update"])
report(result["oval_id"], result["minimum_version"])
elif args["path"] and not os.path.isfile(args["path"]):
message("info", "Definition fragment path not found: " + args["path"])
sys.exit(0)
else:
result = determine_def_min_version(args["path"], definitions_index, elements_index, args["update"])
report(args["path"], result["minimum_version"])
seconds_elapsed = time.time() - start_time
report("info", "Completed in {0}!".format(format_duration(seconds_elapsed)))
