def update(self, force_rebuild=False):
""" Adds/updates all items in repo to index. Note: querying will call this automatically."""
# if we've already updated the index during this script run, we're done!
if self.index_updated:
return False
# if the index is not based on the current commit, rebuild from scratch
if not self.index_based_on_current_commit():
force_rebuild = True
if force_rebuild:
# get a new clean/empty index
index = self.get_index(force_rebuild)
index_writer = index.writer()
# index all documents
documents = self.document_iterator()
activity_description = 'Rebuilding'
else:
# use the current index
index = self.get_index()
index_writer = index.writer()
# delete uncommitted files that are in index already
for filepath in self.get_indexed_uncommitted_files():
index_writer.delete_by_term('path', filepath)
# get list of uncommitted files and persist it
uncommitted_files = lib_git.get_uncommitted_oval()
self.set_indexed_uncommitted_files(uncommitted_files)
# if there are no uncommitted files to index, we're done
if not uncommitted_files:
index_writer.commit()
return False
# index only uncommitted files
documents = self.document_iterator(uncommitted_files)
activity_description = 'Updating'
# add all definition files to index
counter = 0
for document in documents:
counter = counter + 1
self.status_spinner(counter, '{0} {1} index'.format(activity_description, self.index_name), self.item_label)
if 'deleted' in document and document['deleted']:
index_writer.delete_by_term('path', document['path'])
#self.message('debug', 'Deleting from index:\n\t{0} '.format(document['path']))
else:
index_writer.add_document(**document)
#self.message('debug', 'Upserting to index:\n\t{0} '.format(document['path']))
index_writer.commit()
self.status_spinner(counter, '{0} {1} index'.format(activity_description, self.index_name), self.item_label, True)
# update indexed commit
self.set_indexed_commit_hash()
self.index_updated = True
def get_changed_ids_from_git():
try:
changed_files = lib_git.get_uncommitted_oval()
if not changed_files or changed_files is None:
return None
change_list = []
for file in changed_files:
change_list.add(lib_repo.path_to_oval_id(file))
return change_list
except Exception:
if main.verbose:
print("## Error while querying git for changes: ", format(Exception))
return None
def update(self, force_rebuild=False):
""" Adds/updates all items in repo to index. Note: querying will call this automatically."""
# if we've already updated the index during this script run, we're done!
if self.index_updated:
return False
# we only need to do this once per script lifetime
self.index_updated = True
# if the index is not based on the current commit, rebuild from scratch
if not self.index_based_on_current_commit():
force_rebuild = True
if force_rebuild:
# get a new clean/empty index
index = self.get_index(force_rebuild)
# get a high-performance writer as per: https://pythonhosted.org/Whoosh/batch.html
index_writer = index.writer(procs=4, multisegment=True)
# index all documents
documents = self.document_iterator()
activity_description = 'Rebuilding'
# update indexed commit
self.set_indexed_commit_hash()
else:
# use the current index
index = self.get_index()
index_writer = index.writer()
# delete uncommitted files that are in index already
for filepath in self.get_indexed_uncommitted_files():
index_writer.delete_by_term('path', filepath)
# get list of uncommitted files and persist it
uncommitted_files = lib_git.get_uncommitted_oval()
self.set_indexed_uncommitted_files(uncommitted_files)
# nothing to update? done!
if not uncommitted_files:
index_writer.commit()
return
# index only uncommitted files
documents = self.document_iterator(uncommitted_files)
activity_description = 'Updating'
# add all definition files to index
counter = 0
try:
for document in documents:
counter = counter + 1
self.status_spinner(counter, '{0} {1} index'.format(activity_description, self.index_name), self.item_label)
if 'deleted' in document and document['deleted']:
index_writer.delete_by_term('oval_id', self.whoosh_escape(document['oval_id']))
#self.message('debug', 'Deleting from index:\n\t{0} '.format(self.whoosh_escape(document['oval_id'])))
else:
index_writer.add_document(**document)
#self.message('debug', 'Upserting to index:\n\t{0} '.format(document['path']))
except lib_xml.InvalidXmlError as e:
# abort: cannot build index
self.message('ERROR CANNOT BUILD INDEX', 'Invalid xml fragment\n\tFile: {0}\n\tMessage: {1}'.format(e.path, e.message))
self.message('ERROR', 'deleting index and aborting execution')
index_writer.commit()
self.index.close()
shutil.rmtree(self.get_index_path())
sys.exit(1)
self.status_spinner(counter, '{0} {1} index'.format(activity_description, self.index_name), self.item_label, True)
index_writer.commit()
def main():
SCHEMA_VERSION = '5.11.1'
global verbose
global debug
global autoaccept
parser = argparse.ArgumentParser(description='Performs all of the identified QA tasks against an OVAL submission in the repository')
output_options = parser.add_argument_group('verbosity options')
output_options.add_argument('-v', '--verbose', default=False, action="store_true", help='Verbose progress messages')
output_options.add_argument('-d', '--debug', default=False, action="store_true", help='Include debug information on errors')
# output_options.add_argument('-a', '--autoaccept', default=False, action="store_true", help='Automatically continue without prompting when it is safe to do so')
args = vars(parser.parse_args())
if args['verbose']:
verbose = True
if args['debug']:
debug = True
# if args['autoaccept']:
# autoaccept = True
# Grab some things we're going to need later
# First, build the schema path cache
element_index = lib_search.ElementsIndex(message)
schema_path_cache = {}
for schema_version in lib_repo.get_schema_versions():
schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)
# 1. Locate all uncommitted changes to the local repository
if verbose:
print("\n + 1: looking for uncommitted changes")
change_list = lib_git.get_uncommitted_oval()
if change_list is None or len(change_list) < 1:
print("\n-------- This update does not include any changes of significance --------\n")
return
# 1.1 Determine which of these changes are due to removed files
if verbose:
print(" +++ Number of changed items to QA: {0}".format(len(change_list)))
remove_list = find_removed_items(change_list)
# 1.2 Issue warning (prompt to continue) if any of the changes are a deleted item
if verbose:
print(" + 1.2: Determining if any changes are deleted items...")
if remove_list is not None and len(remove_list) > 0:
print("\n -------- The following files were removed as a part of this update --------\n")
show_files(remove_list)
# TODO: Offer the option to inspect the OVALIDs in the removed files and
# build a list of what items, if any, refer to them
response = input("\n :: Accept these changes (N[o] / y[es] / s[how affected]): ")
if response == 's' or response == 'S':
show_affected(remove_list)
response = input("\n :::: Accept these changes (N[o] / y[es]): ")
if response != 'y' and response != 'Y':
return
elif response != 'y' and response != 'Y':
return
# 1.3 Don't include removed files as part of the update
change_list = [file for file in change_list if file not in remove_list]
elif verbose:
print(" +++ No removed items found")
# 2. Remove all changes that are semantically the same as existing elements (except for states)
if verbose:
print("\n + 2: Removing items that don't contain meaningful changes...")
change_list = prune_unchanged_elements(change_list)
# 2.1 If that means we have no changes left, there is nothing else to do
if len(change_list) < 1:
print("\n ----- This update does not include any changes of significance")
return
print("\n ---- Number of changed elements to process: {0}\n".format(len(change_list)))
# 3. For each element in the list that is a definition, check:
if verbose:
print(" + 3: Checking correctness of definition metadata")
def_list = [ path for path in change_list if lib_repo.get_element_type_from_path(path) == 'definition']
if def_list is not None and len(def_list) > 0:
valid_metadata = 1
if verbose:
print(" +++ Number of definitions in this update: {0}".format(len(def_list)))
for def_path in def_list:
def_element = lib_xml.load_standalone_element(def_path)
ode = lib_oval.OvalElement(def_element)
od = lib_oval.OvalDefinition(ode.getElement())
def_id = od.getId()
# 3.1 If this is an update, does it change any existing metadata?
# 3.2 Check existence and accuracy of definition metadata (<status> and date)
# - INITIAL SUBMISSION or DRAFT on new submission
# - INTERIM if updating a previous definition
# - ?
# no <dates> - invalid
# @version == 0:
# no <submitted> - invalid
# <status_change>s > 0 - invalid
# <status> != "INITIAL SUBMISSION" - invalid
# @ version > 0:
# last <status_change> != <status> - invalid
def_status_change = od.get_last_status_change()
if def_status_change["Version"] == "0":
if "Submitted" not in def_status_change or def_status_change["Submitted"] is None:
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions must contain a submitted element")
valid_metadata = 0
if def_status_change["StatusChange"]:
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions should not contain a status change element")
valid_metadata = 0
if def_status_change["Status"] != "INITIAL SUBMISSION":
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions must have a status of INITIAL SUBMISSION")
valid_metadata = 0
else:
defstatus = def_status_change["Status"]
lscstatus = def_status_change["StatusChange"]["Status"]
if (defstatus != lscstatus):
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - Last status change (%s) does not match definition status (%s)" % (lscstatus, defstatus))
valid_metadata = 0
if valid_metadata == 0:
print("\n ++++ Definition Metadata is Invalid. Exiting...")
return
elif verbose:
print(" +++ No definitions to check")
# 4. Schema validate the changes
# First, generate an OVAL document
if verbose:
print("\n + 4: Schema validating changes...")
schema_path = lib_repo.get_oval_def_schema(SCHEMA_VERSION)
for element_file in change_list:
try:
lib_xml.schema_validate(element_file, schema_path, True)
except Exception as e:
print(' Schema validation failed:\n\t{0}'.format(e.message))
print("\n ### Offending file {0}".format(element_file))
return
if verbose:
print(" ++++ Schema validations passed")
print("\n + 5: Updating elements...")
# 5. On passing all of the above, make these changes for all elements:
oval_id_map = {}
affected_elements = set()
update_elements = {}
for path in change_list:
oval_element = lib_xml.load_standalone_element(path)
update_elements[path] = oval_element
# 5.1 If it's a definition, determine and set the minimum schema version
ovalid = oval_element.get("id")
if verbose:
print("\n ---- Processing submitted element {0}".format(ovalid))
if lib_repo.get_element_type_from_path(path) == 'definition':
if verbose:
print(" --- Is a definition: determining minimum schema version")
# min_schema = determine_definition_minimum_schema_version(path, element_index, schema_path_cache)
min_schema = determine_definition_minimum_schema_version(oval_element, element_index, schema_path_cache)
if min_schema and min_schema is not None:
if verbose:
print(" ---- Determined minimum schema version to be {0}".format(min_schema))
set_minimum_schema_version(oval_element, min_schema)
# 5.2 For each element that is not using an OVALID in the CIS namespace:
is_update = True
if not is_repository_id(ovalid):
is_update = False
element_type = lib_repo.get_element_type_from_path(path)
new_id = generate_next_ovalid(element_type, element_index)
if verbose:
print(" ---- Change submission ID from '{0}' to '{1}'".format(ovalid, new_id))
oval_element.set("id", new_id)
# 5.2.1 Set to a unique OVALID in the CIS namespace
# 5.2.2 Update all references from the old OVALID
oval_id_map[ovalid] = new_id
# 5.3 Set/update version numbers as necessary. The previous step can be used to determine new vice update
if is_update:
# 5.3.1 If this is an update, find the current version and increment by one
if verbose:
print(" ---- Is an update: incrementing version")
increment_version(oval_element)
# Find all upstream elements and add them, as unique, to the list of items to change
if lib_repo.get_element_type_from_path(path) != 'definition':
if verbose:
print(" ---- Not a definition. Finding affected upstream elements...")
affected = find_affected(ovalid, element_index)
if affected is not None and len(affected) > 0:
if verbose:
print(" ---- Number of elements affected: {0}".format(len(affected)))
affected_elements = set().union(affected_elements, affected)
else:
if verbose:
print(" >>>>> Warning: found no affected elements for this update. Possible orphan.")
else:
# Otherwise, set it to 1
oval_element.set("version", "1")
# 5.4 Canonicalize all altered elements (if possible)
# Now that we know all the elements affected by an update we can increment their IDs once
if len(affected_elements) > 0:
if verbose:
print("\n ------- This update affects {0} upstream elements: incrementing the version number for each...".format(len(affected_elements)))
for file in affected_elements:
oval_element = lib_xml.load_standalone_element(file)
if oval_element is not None:
increment_version(oval_element)
#oval_element = normalize_ids(oval_element, oval_id_map)
update_elements[file] = oval_element
# 6 Write the element, and remove the old if the path changed
print("\n=============== Complete ===================")
print("All automated checks have completed successfully, but the following")
print(" manual checks need to be made prior to accepting this submission:")
print(" * Metadata for definitions is complete and accurate")
print(" * Existing metadata has not been changed")
print(" * Contains a meaningful change")
print(" * Does not contain any harmful actions or unacceptable language")
for x in oval_id_map:
print(" -- Convert %s to %s" % (x, oval_id_map[x]))
response = input("\n :::: Save all changes now? (N[o] / y[es]): ")
if response != 'y' and response != 'Y':
return
for path in update_elements:
oval_element = normalize_ids(update_elements[path], oval_id_map)
if not oval_element or oval_element is None:
continue
new_path = lib_repo.get_element_repository_path(oval_element)
if verbose:
print("## Writing {0}".format(new_path))
save_element(oval_element, new_path)
if new_path != path:
if verbose:
print("### Deleting {0}".format(path))
try:
os.remove(path)
except Exception:
print("#### Exception/Skipping Deleting {0}".format(path))
# 7. Prompt for a message to use for the commit
# 7.1 Commit and push the changes
return
def main():
SCHEMA_VERSION = '5.11.1'
global verbose
global debug
global autoaccept
parser = argparse.ArgumentParser(description='Performs all of the identified QA tasks against an OVAL submission in the repository')
output_options = parser.add_argument_group('verbosity options')
output_options.add_argument('-v', '--verbose', default=False, action="store_true", help='Verbose progress messages')
output_options.add_argument('-d', '--debug', default=False, action="store_true", help='Include debug information on errors')
# output_options.add_argument('-a', '--autoaccept', default=False, action="store_true", help='Automatically continue without prompting when it is safe to do so')
args = vars(parser.parse_args())
if args['verbose']:
verbose = True
if args['debug']:
debug = True
# if args['autoaccept']:
# autoaccept = True
# Grab some things we're going to need later
# First, build the schema path cache
element_index = lib_search.ElementsIndex(message)
schema_path_cache = {}
for schema_version in lib_repo.get_schema_versions():
schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)
# 1. Locate all uncommitted changes to the local repository
if verbose:
print("\n + 1: looking for uncommitted changes")
change_list = lib_git.get_uncommitted_oval()
if change_list is None or len(change_list) < 1:
print("\n-------- This update does not include any changes of significance --------\n")
return
# 1.1 Determine which of these changes are due to removed files
if verbose:
print(" +++ Number of changed items to QA: {0}".format(len(change_list)))
remove_list = find_removed_items(change_list)
# 1.2 Issue warning (prompt to continue) if any of the changes are a deleted item
if verbose:
print(" + 1.2: Determining if any changes are deleted items...")
if remove_list is not None and len(remove_list) > 0:
print("\n -------- The following files were removed as a part of this update --------\n")
show_files(remove_list)
# TODO: Offer the option to inspect the OVALIDs in the removed files and
# build a list of what items, if any, refer to them
response = input("\n :: Accept these changes (N[o] / y[es] / s[how affected]): ")
if response == 's' or response == 'S':
show_affected(remove_list)
response = input("\n :::: Accept these changes (N[o] / y[es]): ")
if response != 'y' and response != 'Y':
return
elif response != 'y' and response != 'Y':
return
# 1.3 Don't include removed files as part of the update
change_list = [file for file in change_list if file not in remove_list]
elif verbose:
print(" +++ No removed items found")
# 2. Remove all changes that are semantically the same as existing elements (except for states)
if verbose:
print("\n + 2: Removing items that don't contain meaningful changes...")
change_list = prune_unchanged_elements(change_list)
# 2.1 If that means we have no changes left, there is nothing else to do
if len(change_list) < 1:
print("\n ----- This update does not include any changes of significance")
return
print("\n ---- Number of changed elements to process: {0}\n".format(len(change_list)))
# 3. For each element in the list that is a definition, check:
if verbose:
print(" + 3: Checking correctness of definition metadata")
def_list = [ path for path in change_list if lib_repo.get_element_type_from_path(path) == 'definition']
if def_list is not None and len(def_list) > 0:
valid_metadata = 1
if verbose:
print(" +++ Number of definitions in this update: {0}".format(len(def_list)))
for def_path in def_list:
def_element = lib_xml.load_standalone_element(def_path)
ode = lib_oval.OvalElement(def_element)
od = lib_oval.OvalDefinition(ode.getElement())
def_id = od.getId()
# 3.1 If this is an update, does it change any existing metadata?
# 3.2 Check existence and accuracy of definition metadata (<status> and date)
# - INITIAL SUBMISSION or DRAFT on new submission
# - INTERIM if updating a previous definition
# - ?
# no <dates> - invalid
# @version == 0:
# no <submitted> - invalid
# <status_change>s > 0 - invalid
# <status> != "INITIAL SUBMISSION" - invalid
# @ version > 0:
# last <status_change> != <status> - invalid
def_status_change = od.get_last_status_change()
if def_status_change["Version"] == "0":
if "Submitted" not in def_status_change or def_status_change["Submitted"] is None:
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions must contain a submitted element")
valid_metadata = 0
if def_status_change["StatusChange"]:
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions should not contain a status change element")
valid_metadata = 0
if def_status_change["Status"] != "INITIAL SUBMISSION":
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions must have a status of INITIAL SUBMISSION")
valid_metadata = 0
else:
defstatus = def_status_change["Status"]
lscstatus = def_status_change["StatusChange"]["Status"]
if (defstatus != lscstatus):
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - Last status change (%s) does not match definition status (%s)" % (lscstatus, defstatus))
valid_metadata = 0
if valid_metadata == 0:
print("\n ++++ Definition Metadata is Invalid. Exiting...")
return
elif verbose:
print(" +++ No definitions to check")
# 4. Schema validate the changes
# First, generate an OVAL document
if verbose:
print("\n + 4: Schema validating changes...")
schema_path = lib_repo.get_oval_def_schema(SCHEMA_VERSION)
for element_file in change_list:
try:
lib_xml.schema_validate(element_file, schema_path, True)
except Exception as e:
print(' Schema validation failed:\n\t{0}'.format(e.message))
print("\n ### Offending file {0}".format(element_file))
return
if verbose:
print(" ++++ Schema validations passed")
print("\n + 5: Updating elements...")
# 5. On passing all of the above, make these changes for all elements:
oval_id_map = {}
affected_elements = set()
update_elements = {}
for path in change_list:
oval_element = lib_xml.load_standalone_element(path)
update_elements[path] = oval_element
# 5.1 If it's a definition, determine and set the minimum schema version
ovalid = oval_element.get("id")
if verbose:
print("\n ---- Processing submitted element {0}".format(ovalid))
if lib_repo.get_element_type_from_path(path) == 'definition':
if verbose:
print(" --- Is a definition: determining minimum schema version")
# min_schema = determine_definition_minimum_schema_version(path, element_index, schema_path_cache)
min_schema = determine_definition_minimum_schema_version(oval_element, element_index, schema_path_cache)
if min_schema and min_schema is not None:
if verbose:
print(" ---- Determined minimum schema version to be {0}".format(min_schema))
set_minimum_schema_version(oval_element, min_schema)
# 5.2 For each element that is not using an OVALID in the CIS namespace:
is_update = True
if not is_repository_id(ovalid):
is_update = False
element_type = lib_repo.get_element_type_from_path(path)
new_id = generate_next_ovalid(element_type, element_index)
if verbose:
print(" ---- Change submission ID from '{0}' to '{1}'".format(ovalid, new_id))
oval_element.set("id", new_id)
# 5.2.1 Set to a unique OVALID in the CIS namespace
# 5.2.2 Update all references from the old OVALID
oval_id_map[ovalid] = new_id
# 5.3 Set/update version numbers as necessary. The previous step can be used to determine new vice update
if is_update:
# 5.3.1 If this is an update, find the current version and increment by one
if verbose:
print(" ---- Is an update: incrementing version")
increment_version(oval_element)
# Find all upstream elements and add them, as unique, to the list of items to change
if lib_repo.get_element_type_from_path(path) != 'definition':
if verbose:
print(" ---- Not a definition. Finding affected upstream elements...")
affected = find_affected(ovalid, element_index)
if affected is not None and len(affected) > 0:
if verbose:
print(" ---- Number of elements affected: {0}".format(len(affected)))
affected_elements = set().union(affected_elements, affected)
else:
if verbose:
print(" >>>>> Warning: found no affected elements for this update. Possible orphan.")
else:
# Otherwise, set it to 1
oval_element.set("version", "1")
# 5.4 Canonicalize all altered elements (if possible)
# Now that we know all the elements affected by an update we can increment their IDs once
if len(affected_elements) > 0:
if verbose:
print("\n ------- This update affects {0} upstream elements: incrementing the version number for each...".format(len(affected_elements)))
for file in affected_elements:
oval_element = lib_xml.load_standalone_element(file)
if oval_element is not None:
increment_version(oval_element)
#oval_element = normalize_ids(oval_element, oval_id_map)
update_elements[file] = oval_element
# 6 Write the element, and remove the old if the path changed
print("\n=============== Complete ===================")
print("All automated checks have completed successfully, but the following")
print(" manual checks need to be made prior to accepting this submission:")
print(" * Metadata for definitions is complete and accurate")
print(" * Existing metadata has not been changed")
print(" * Contains a meaningful change")
print(" * Does not contain any harmful actions or unacceptable language")
for x in oval_id_map:
print(" -- Convert %s to %s" % (x, oval_id_map[x]))
response = input("\n :::: Save all changes now? (N[o] / y[es]): ")
if response != 'y' and response != 'Y':
return
for path in update_elements:
oval_element = normalize_ids(update_elements[path], oval_id_map)
if not oval_element or oval_element is None:
continue
new_path = lib_repo.get_element_repository_path(oval_element)
if verbose:
print("## Writing {0}".format(new_path))
save_element(oval_element, new_path)
if new_path != path:
if verbose:
print("### Deleting {0}".format(path))
os.remove(path)
# 7. Prompt for a message to use for the commit
# 7.1 Commit and push the changes
return
def update(self, force_rebuild=False):
""" Adds/updates all items in repo to index. Note: querying will call this automatically."""
# if we've already updated the index during this script run, we're done!
if self.index_updated:
return False
# we only need to do this once per script lifetime
self.index_updated = True
# if the index is not based on the current commit, rebuild from scratch
if not self.index_based_on_current_commit():
force_rebuild = True
if force_rebuild:
# get a new clean/empty index
index = self.get_index(force_rebuild)
# disabled high-performance writer (https://pythonhosted.org/Whoosh/batch.html), causing thread/lock issues
# index_writer = index.writer(procs=4, multisegment=True)
index_writer = index.writer()
# index all documents
documents = self.document_iterator()
activity_description = 'Rebuilding'
# update indexed commit
self.set_indexed_commit_hash()
else:
# use the current index
index = self.get_index()
index_writer = index.writer()
# delete uncommitted files that are in index already
for filepath in self.get_indexed_uncommitted_files():
index_writer.delete_by_term('path', filepath)
# get list of uncommitted files and persist it
uncommitted_files = lib_git.get_uncommitted_oval()
self.set_indexed_uncommitted_files(uncommitted_files)
# nothing to update? done!
if not uncommitted_files:
index_writer.commit()
return
# index only uncommitted files
documents = self.document_iterator(uncommitted_files)
activity_description = 'Updating'
# add all definition files to index
counter = 0
try:
for document in documents:
counter = counter + 1
self.status_spinner(
counter, '{0} {1} index'.format(activity_description,
self.index_name),
self.item_label)
if 'deleted' in document and document['deleted']:
try:
index_writer.delete_by_term(
'oval_id', self.whoosh_escape(document['oval_id']))
except:
self.message(
'debug',
'Something was marked as needing to be deleted but it wasnt in the index'
)
#self.message('debug', 'Deleting from index:\n\t{0} '.format(self.whoosh_escape(document['oval_id'])))
#index_writer.delete_by_term('oval_id', self.whoosh_escape(document['oval_id']))
#self.message('debug', 'Deleting from index:\n\t{0} '.format(self.whoosh_escape(document['oval_id'])))
else:
index_writer.add_document(**document)
#self.message('debug', 'Upserting to index:\n\t{0} '.format(document['path']))
except lib_xml.InvalidXmlError as e:
# abort: cannot build index
self.message(
'ERROR CANNOT BUILD INDEX',
'Invalid xml fragment\n\tFile: {0}\n\tMessage: {1}'.format(
e.path, e.message))
self.message('ERROR', 'deleting index and aborting execution')
index_writer.commit()
self.index.close()
shutil.rmtree(self.get_index_path())
sys.exit(1)
self.status_spinner(
counter, '{0} {1} index'.format(activity_description,
self.index_name), self.item_label,
True)
index_writer.commit()