def writeFiles(element_list, repo_root, verbose=False):
if not element_list or element_list is None:
return
for element in element_list:
e = element.getElement()
filepath = lib_repo.get_element_repository_path(e)
if filepath and filepath is not None:
writeFile(filepath, element, verbose)
def writeFiles(element_list, repo_root, verbose=False):
if not element_list or element_list is None:
return
for element in element_list:
e = element.getElement()
filepath = lib_repo.get_element_repository_path(e)
if filepath and filepath is not None:
writeFile(filepath, element, verbose)
def process_elements(element_list):
if element_list is None:
return True
global verbose
global force
global schema_path
file_list = []
for element in element_list:
ovalid = element.get("id")
filepath = lib_repo.get_element_repository_path(element)
if not filepath or filepath is None:
if verbose:
print(
" ### Unable to determine repository location for '{0}'".
format(ovalid))
return False
if os.path.isfile(filepath):
if verbose:
print(
" ++ Element '{0}' already exists. Checking for changes..."
.format(ovalid))
original = lib_xml.load_standalone_element(filepath)
if original is not None:
changes = lib_xml.get_element_change_status(element, original)
if changes == 2:
print(" +++ Changes found: updating")
else:
if verbose:
print(" +++ No meaningful changes: skipping".
format(ovalid))
continue
elif verbose:
print(
" ++++ New element '{0}':\n ---> adding to repository as '{1}"
.format(ovalid, filepath))
#Write the element to its own file
file_list.append(filepath)
save_element(element, filepath)
print("\n +++ Files added or updated by this submission:")
for file in file_list:
print(" ---> {0}".format(file))
return True
def process_elements(element_list):
if element_list is None:
return True
global verbose
global force
global schema_path
file_list = []
for element in element_list:
ovalid = element.get("id")
filepath = lib_repo.get_element_repository_path(element)
if not filepath or filepath is None:
if verbose:
print(" ### Unable to determine repository location for '{0}'".format(ovalid))
return False
if os.path.isfile(filepath):
if verbose:
print(" ++ Element '{0}' already exists. Checking for changes...".format(ovalid))
original = lib_xml.load_standalone_element(filepath)
if original is not None:
changes = lib_xml.get_element_change_status(element, original)
if changes == 2:
print(" +++ Changes found: updating")
else:
if verbose:
print(" +++ No meaningful changes: skipping".format(ovalid))
continue
elif verbose:
print(" ++++ New element '{0}':\n ---> adding to repository as '{1}".format(ovalid, filepath))
#Write the element to its own file
file_list.append(filepath)
save_element(element, filepath)
print("\n +++ Files added or updated by this submission:")
for file in file_list:
print(" ---> {0}".format(file))
return True
def main():
print("\n--- Starting tests...\n")
tree = etree.parse('change_tests.xml')
if tree is None:
print("ERROR", "Unable to parse test definition file into XML")
return
root = tree.getroot()
if root is None:
print("ERROR", "Could not locate document root")
return
test_cases = list(root)
for element in test_cases:
# Get the corresponding element from the repository
ovalid = element.get('id')
path = lib_repo.get_element_repository_path(element)
existing = lib_xml.load_standalone_element(path)
status = lib_xml.get_element_change_status(element, existing)
print(" ++ Status for element {0} = {1}".format(ovalid, status))
def main():
print("\n--- Starting tests...\n")
tree = etree.parse('change_tests.xml')
if tree is None:
print("ERROR", "Unable to parse test definition file into XML")
return
root = tree.getroot()
if root is None:
print("ERROR", "Could not locate document root")
return
test_cases = list(root)
for element in test_cases:
# Get the corresponding element from the repository
ovalid = element.get('id')
path = lib_repo.get_element_repository_path(element)
existing = lib_xml.load_standalone_element(path)
status = lib_xml.get_element_change_status(element, existing)
print(" ++ Status for element {0} = {1}".format(ovalid, status))
def main():
SCHEMA_VERSION = '5.11.1'
global verbose
global debug
global autoaccept
parser = argparse.ArgumentParser(description='Performs all of the identified QA tasks against an OVAL submission in the repository')
output_options = parser.add_argument_group('verbosity options')
output_options.add_argument('-v', '--verbose', default=False, action="store_true", help='Verbose progress messages')
output_options.add_argument('-d', '--debug', default=False, action="store_true", help='Include debug information on errors')
# output_options.add_argument('-a', '--autoaccept', default=False, action="store_true", help='Automatically continue without prompting when it is safe to do so')
args = vars(parser.parse_args())
if args['verbose']:
verbose = True
if args['debug']:
debug = True
# if args['autoaccept']:
# autoaccept = True
# Grab some things we're going to need later
# First, build the schema path cache
element_index = lib_search.ElementsIndex(message)
schema_path_cache = {}
for schema_version in lib_repo.get_schema_versions():
schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)
# 1. Locate all uncommitted changes to the local repository
if verbose:
print("\n + 1: looking for uncommitted changes")
change_list = lib_git.get_uncommitted_oval()
if change_list is None or len(change_list) < 1:
print("\n-------- This update does not include any changes of significance --------\n")
return
# 1.1 Determine which of these changes are due to removed files
if verbose:
print(" +++ Number of changed items to QA: {0}".format(len(change_list)))
remove_list = find_removed_items(change_list)
# 1.2 Issue warning (prompt to continue) if any of the changes are a deleted item
if verbose:
print(" + 1.2: Determining if any changes are deleted items...")
if remove_list is not None and len(remove_list) > 0:
print("\n -------- The following files were removed as a part of this update --------\n")
show_files(remove_list)
# TODO: Offer the option to inspect the OVALIDs in the removed files and
# build a list of what items, if any, refer to them
response = input("\n :: Accept these changes (N[o] / y[es] / s[how affected]): ")
if response == 's' or response == 'S':
show_affected(remove_list)
response = input("\n :::: Accept these changes (N[o] / y[es]): ")
if response != 'y' and response != 'Y':
return
elif response != 'y' and response != 'Y':
return
# 1.3 Don't include removed files as part of the update
change_list = [file for file in change_list if file not in remove_list]
elif verbose:
print(" +++ No removed items found")
# 2. Remove all changes that are semantically the same as existing elements (except for states)
if verbose:
print("\n + 2: Removing items that don't contain meaningful changes...")
change_list = prune_unchanged_elements(change_list)
# 2.1 If that means we have no changes left, there is nothing else to do
if len(change_list) < 1:
print("\n ----- This update does not include any changes of significance")
return
print("\n ---- Number of changed elements to process: {0}\n".format(len(change_list)))
# 3. For each element in the list that is a definition, check:
if verbose:
print(" + 3: Checking correctness of definition metadata")
def_list = [ path for path in change_list if lib_repo.get_element_type_from_path(path) == 'definition']
if def_list is not None and len(def_list) > 0:
valid_metadata = 1
if verbose:
print(" +++ Number of definitions in this update: {0}".format(len(def_list)))
for def_path in def_list:
def_element = lib_xml.load_standalone_element(def_path)
ode = lib_oval.OvalElement(def_element)
od = lib_oval.OvalDefinition(ode.getElement())
def_id = od.getId()
# 3.1 If this is an update, does it change any existing metadata?
# 3.2 Check existence and accuracy of definition metadata (<status> and date)
# - INITIAL SUBMISSION or DRAFT on new submission
# - INTERIM if updating a previous definition
# - ?
# no <dates> - invalid
# @version == 0:
# no <submitted> - invalid
# <status_change>s > 0 - invalid
# <status> != "INITIAL SUBMISSION" - invalid
# @ version > 0:
# last <status_change> != <status> - invalid
def_status_change = od.get_last_status_change()
if def_status_change["Version"] == "0":
if "Submitted" not in def_status_change or def_status_change["Submitted"] is None:
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions must contain a submitted element")
valid_metadata = 0
if def_status_change["StatusChange"]:
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions should not contain a status change element")
valid_metadata = 0
if def_status_change["Status"] != "INITIAL SUBMISSION":
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions must have a status of INITIAL SUBMISSION")
valid_metadata = 0
else:
defstatus = def_status_change["Status"]
lscstatus = def_status_change["StatusChange"]["Status"]
if (defstatus != lscstatus):
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - Last status change (%s) does not match definition status (%s)" % (lscstatus, defstatus))
valid_metadata = 0
if valid_metadata == 0:
print("\n ++++ Definition Metadata is Invalid. Exiting...")
return
elif verbose:
print(" +++ No definitions to check")
# 4. Schema validate the changes
# First, generate an OVAL document
if verbose:
print("\n + 4: Schema validating changes...")
schema_path = lib_repo.get_oval_def_schema(SCHEMA_VERSION)
for element_file in change_list:
try:
lib_xml.schema_validate(element_file, schema_path, True)
except Exception as e:
print(' Schema validation failed:\n\t{0}'.format(e.message))
print("\n ### Offending file {0}".format(element_file))
return
if verbose:
print(" ++++ Schema validations passed")
print("\n + 5: Updating elements...")
# 5. On passing all of the above, make these changes for all elements:
oval_id_map = {}
affected_elements = set()
update_elements = {}
for path in change_list:
oval_element = lib_xml.load_standalone_element(path)
update_elements[path] = oval_element
# 5.1 If it's a definition, determine and set the minimum schema version
ovalid = oval_element.get("id")
if verbose:
print("\n ---- Processing submitted element {0}".format(ovalid))
if lib_repo.get_element_type_from_path(path) == 'definition':
if verbose:
print(" --- Is a definition: determining minimum schema version")
# min_schema = determine_definition_minimum_schema_version(path, element_index, schema_path_cache)
min_schema = determine_definition_minimum_schema_version(oval_element, element_index, schema_path_cache)
if min_schema and min_schema is not None:
if verbose:
print(" ---- Determined minimum schema version to be {0}".format(min_schema))
set_minimum_schema_version(oval_element, min_schema)
# 5.2 For each element that is not using an OVALID in the CIS namespace:
is_update = True
if not is_repository_id(ovalid):
is_update = False
element_type = lib_repo.get_element_type_from_path(path)
new_id = generate_next_ovalid(element_type, element_index)
if verbose:
print(" ---- Change submission ID from '{0}' to '{1}'".format(ovalid, new_id))
oval_element.set("id", new_id)
# 5.2.1 Set to a unique OVALID in the CIS namespace
# 5.2.2 Update all references from the old OVALID
oval_id_map[ovalid] = new_id
# 5.3 Set/update version numbers as necessary. The previous step can be used to determine new vice update
if is_update:
# 5.3.1 If this is an update, find the current version and increment by one
if verbose:
print(" ---- Is an update: incrementing version")
increment_version(oval_element)
# Find all upstream elements and add them, as unique, to the list of items to change
if lib_repo.get_element_type_from_path(path) != 'definition':
if verbose:
print(" ---- Not a definition. Finding affected upstream elements...")
affected = find_affected(ovalid, element_index)
if affected is not None and len(affected) > 0:
if verbose:
print(" ---- Number of elements affected: {0}".format(len(affected)))
affected_elements = set().union(affected_elements, affected)
else:
if verbose:
print(" >>>>> Warning: found no affected elements for this update. Possible orphan.")
else:
# Otherwise, set it to 1
oval_element.set("version", "1")
# 5.4 Canonicalize all altered elements (if possible)
# Now that we know all the elements affected by an update we can increment their IDs once
if len(affected_elements) > 0:
if verbose:
print("\n ------- This update affects {0} upstream elements: incrementing the version number for each...".format(len(affected_elements)))
for file in affected_elements:
oval_element = lib_xml.load_standalone_element(file)
if oval_element is not None:
increment_version(oval_element)
#oval_element = normalize_ids(oval_element, oval_id_map)
update_elements[file] = oval_element
# 6 Write the element, and remove the old if the path changed
print("\n=============== Complete ===================")
print("All automated checks have completed successfully, but the following")
print(" manual checks need to be made prior to accepting this submission:")
print(" * Metadata for definitions is complete and accurate")
print(" * Existing metadata has not been changed")
print(" * Contains a meaningful change")
print(" * Does not contain any harmful actions or unacceptable language")
for x in oval_id_map:
print(" -- Convert %s to %s" % (x, oval_id_map[x]))
response = input("\n :::: Save all changes now? (N[o] / y[es]): ")
if response != 'y' and response != 'Y':
return
for path in update_elements:
oval_element = normalize_ids(update_elements[path], oval_id_map)
if not oval_element or oval_element is None:
continue
new_path = lib_repo.get_element_repository_path(oval_element)
if verbose:
print("## Writing {0}".format(new_path))
save_element(oval_element, new_path)
if new_path != path:
if verbose:
print("### Deleting {0}".format(path))
try:
os.remove(path)
except Exception:
print("#### Exception/Skipping Deleting {0}".format(path))
# 7. Prompt for a message to use for the commit
# 7.1 Commit and push the changes
return
def main():
SCHEMA_VERSION = '5.11.1'
global verbose
global debug
global autoaccept
parser = argparse.ArgumentParser(description='Performs all of the identified QA tasks against an OVAL submission in the repository')
output_options = parser.add_argument_group('verbosity options')
output_options.add_argument('-v', '--verbose', default=False, action="store_true", help='Verbose progress messages')
output_options.add_argument('-d', '--debug', default=False, action="store_true", help='Include debug information on errors')
# output_options.add_argument('-a', '--autoaccept', default=False, action="store_true", help='Automatically continue without prompting when it is safe to do so')
args = vars(parser.parse_args())
if args['verbose']:
verbose = True
if args['debug']:
debug = True
# if args['autoaccept']:
# autoaccept = True
# Grab some things we're going to need later
# First, build the schema path cache
element_index = lib_search.ElementsIndex(message)
schema_path_cache = {}
for schema_version in lib_repo.get_schema_versions():
schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)
# 1. Locate all uncommitted changes to the local repository
if verbose:
print("\n + 1: looking for uncommitted changes")
change_list = lib_git.get_uncommitted_oval()
if change_list is None or len(change_list) < 1:
print("\n-------- This update does not include any changes of significance --------\n")
return
# 1.1 Determine which of these changes are due to removed files
if verbose:
print(" +++ Number of changed items to QA: {0}".format(len(change_list)))
remove_list = find_removed_items(change_list)
# 1.2 Issue warning (prompt to continue) if any of the changes are a deleted item
if verbose:
print(" + 1.2: Determining if any changes are deleted items...")
if remove_list is not None and len(remove_list) > 0:
print("\n -------- The following files were removed as a part of this update --------\n")
show_files(remove_list)
# TODO: Offer the option to inspect the OVALIDs in the removed files and
# build a list of what items, if any, refer to them
response = input("\n :: Accept these changes (N[o] / y[es] / s[how affected]): ")
if response == 's' or response == 'S':
show_affected(remove_list)
response = input("\n :::: Accept these changes (N[o] / y[es]): ")
if response != 'y' and response != 'Y':
return
elif response != 'y' and response != 'Y':
return
# 1.3 Don't include removed files as part of the update
change_list = [file for file in change_list if file not in remove_list]
elif verbose:
print(" +++ No removed items found")
# 2. Remove all changes that are semantically the same as existing elements (except for states)
if verbose:
print("\n + 2: Removing items that don't contain meaningful changes...")
change_list = prune_unchanged_elements(change_list)
# 2.1 If that means we have no changes left, there is nothing else to do
if len(change_list) < 1:
print("\n ----- This update does not include any changes of significance")
return
print("\n ---- Number of changed elements to process: {0}\n".format(len(change_list)))
# 3. For each element in the list that is a definition, check:
if verbose:
print(" + 3: Checking correctness of definition metadata")
def_list = [ path for path in change_list if lib_repo.get_element_type_from_path(path) == 'definition']
if def_list is not None and len(def_list) > 0:
valid_metadata = 1
if verbose:
print(" +++ Number of definitions in this update: {0}".format(len(def_list)))
for def_path in def_list:
def_element = lib_xml.load_standalone_element(def_path)
ode = lib_oval.OvalElement(def_element)
od = lib_oval.OvalDefinition(ode.getElement())
def_id = od.getId()
# 3.1 If this is an update, does it change any existing metadata?
# 3.2 Check existence and accuracy of definition metadata (<status> and date)
# - INITIAL SUBMISSION or DRAFT on new submission
# - INTERIM if updating a previous definition
# - ?
# no <dates> - invalid
# @version == 0:
# no <submitted> - invalid
# <status_change>s > 0 - invalid
# <status> != "INITIAL SUBMISSION" - invalid
# @ version > 0:
# last <status_change> != <status> - invalid
def_status_change = od.get_last_status_change()
if def_status_change["Version"] == "0":
if "Submitted" not in def_status_change or def_status_change["Submitted"] is None:
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions must contain a submitted element")
valid_metadata = 0
if def_status_change["StatusChange"]:
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions should not contain a status change element")
valid_metadata = 0
if def_status_change["Status"] != "INITIAL SUBMISSION":
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - New definitions must have a status of INITIAL SUBMISSION")
valid_metadata = 0
else:
defstatus = def_status_change["Status"]
lscstatus = def_status_change["StatusChange"]["Status"]
if (defstatus != lscstatus):
print(" ++++ Definition ID %s is NOT valid:" % def_id)
print(" - Last status change (%s) does not match definition status (%s)" % (lscstatus, defstatus))
valid_metadata = 0
if valid_metadata == 0:
print("\n ++++ Definition Metadata is Invalid. Exiting...")
return
elif verbose:
print(" +++ No definitions to check")
# 4. Schema validate the changes
# First, generate an OVAL document
if verbose:
print("\n + 4: Schema validating changes...")
schema_path = lib_repo.get_oval_def_schema(SCHEMA_VERSION)
for element_file in change_list:
try:
lib_xml.schema_validate(element_file, schema_path, True)
except Exception as e:
print(' Schema validation failed:\n\t{0}'.format(e.message))
print("\n ### Offending file {0}".format(element_file))
return
if verbose:
print(" ++++ Schema validations passed")
print("\n + 5: Updating elements...")
# 5. On passing all of the above, make these changes for all elements:
oval_id_map = {}
affected_elements = set()
update_elements = {}
for path in change_list:
oval_element = lib_xml.load_standalone_element(path)
update_elements[path] = oval_element
# 5.1 If it's a definition, determine and set the minimum schema version
ovalid = oval_element.get("id")
if verbose:
print("\n ---- Processing submitted element {0}".format(ovalid))
if lib_repo.get_element_type_from_path(path) == 'definition':
if verbose:
print(" --- Is a definition: determining minimum schema version")
# min_schema = determine_definition_minimum_schema_version(path, element_index, schema_path_cache)
min_schema = determine_definition_minimum_schema_version(oval_element, element_index, schema_path_cache)
if min_schema and min_schema is not None:
if verbose:
print(" ---- Determined minimum schema version to be {0}".format(min_schema))
set_minimum_schema_version(oval_element, min_schema)
# 5.2 For each element that is not using an OVALID in the CIS namespace:
is_update = True
if not is_repository_id(ovalid):
is_update = False
element_type = lib_repo.get_element_type_from_path(path)
new_id = generate_next_ovalid(element_type, element_index)
if verbose:
print(" ---- Change submission ID from '{0}' to '{1}'".format(ovalid, new_id))
oval_element.set("id", new_id)
# 5.2.1 Set to a unique OVALID in the CIS namespace
# 5.2.2 Update all references from the old OVALID
oval_id_map[ovalid] = new_id
# 5.3 Set/update version numbers as necessary. The previous step can be used to determine new vice update
if is_update:
# 5.3.1 If this is an update, find the current version and increment by one
if verbose:
print(" ---- Is an update: incrementing version")
increment_version(oval_element)
# Find all upstream elements and add them, as unique, to the list of items to change
if lib_repo.get_element_type_from_path(path) != 'definition':
if verbose:
print(" ---- Not a definition. Finding affected upstream elements...")
affected = find_affected(ovalid, element_index)
if affected is not None and len(affected) > 0:
if verbose:
print(" ---- Number of elements affected: {0}".format(len(affected)))
affected_elements = set().union(affected_elements, affected)
else:
if verbose:
print(" >>>>> Warning: found no affected elements for this update. Possible orphan.")
else:
# Otherwise, set it to 1
oval_element.set("version", "1")
# 5.4 Canonicalize all altered elements (if possible)
# Now that we know all the elements affected by an update we can increment their IDs once
if len(affected_elements) > 0:
if verbose:
print("\n ------- This update affects {0} upstream elements: incrementing the version number for each...".format(len(affected_elements)))
for file in affected_elements:
oval_element = lib_xml.load_standalone_element(file)
if oval_element is not None:
increment_version(oval_element)
#oval_element = normalize_ids(oval_element, oval_id_map)
update_elements[file] = oval_element
# 6 Write the element, and remove the old if the path changed
print("\n=============== Complete ===================")
print("All automated checks have completed successfully, but the following")
print(" manual checks need to be made prior to accepting this submission:")
print(" * Metadata for definitions is complete and accurate")
print(" * Existing metadata has not been changed")
print(" * Contains a meaningful change")
print(" * Does not contain any harmful actions or unacceptable language")
for x in oval_id_map:
print(" -- Convert %s to %s" % (x, oval_id_map[x]))
response = input("\n :::: Save all changes now? (N[o] / y[es]): ")
if response != 'y' and response != 'Y':
return
for path in update_elements:
oval_element = normalize_ids(update_elements[path], oval_id_map)
if not oval_element or oval_element is None:
continue
new_path = lib_repo.get_element_repository_path(oval_element)
if verbose:
print("## Writing {0}".format(new_path))
save_element(oval_element, new_path)
if new_path != path:
if verbose:
print("### Deleting {0}".format(path))
os.remove(path)
# 7. Prompt for a message to use for the commit
# 7.1 Commit and push the changes
return
