def main(): """ parse command line options and generate file """ start_time = time.time() # parse command line options parser = argparse.ArgumentParser(description='Get OVAL elements that are related to one or more elements.') parser.add_argument('oval_id', nargs='+', help='one or more OVAL id(s)') direction_options = parser.add_argument_group('direction') direction_options.add_argument('--upstream', default=False, action="store_true", help='find elements that are upstream (i.e. reference) the provided OVAL id(s)') direction_options.add_argument('--downstream', default=False, action="store_true", help='find elements that are downstream (i.e. referenced by) the provided OVAL id(s)') limit_options = parser.add_argument_group('limit element results') limit_options.add_argument('--distance', nargs='?', type=int, help='limit distance from OVAL id(s) (e.g. 1 will only return parents/children, 2 would also return grandparents/grandchildren)') limit_options.add_argument('--element_type', nargs='*', help='filter by element typs(s): {0}'.format(', '.join(lib_repo.supported_element_types))) args = vars(parser.parse_args()) if not (args['upstream'] or args['downstream']): parser.print_help() message('error',"Please specify at least one of: --upstream or --downstream.") sys.exit(0) if args['distance'] and args['distance'] < 1: parser.print_help() message('error','--distance must be a positive integer.') sys.exit(0) if args['element_type']: unsupported_element_types = [ element_type for element_type in args['element_type'] if element_type not in lib_repo.supported_element_types ] if unsupported_element_types: parser.print_help() message('error','Unsupported element type(s): {0}.'.format(', '.join(unsupported_element_types))) sys.exit(0) source_ids = set(args['oval_id']) num_source_ids = len(source_ids) distance = args['distance'] or 0 elements_index = lib_search.ElementsIndex(message) if args['downstream']: # add all downstream element ids message('info','Finding downstream OVAL ids for {0} element(s)'.format(num_source_ids)) downstream_ids = elements_index.find_downstream_ids(source_ids, set(), distance) message('info','Found {0} downstream OVAL ids (all element types)'.format(len(downstream_ids))) if args['element_type']: downstream_ids = { oval_id for oval_id in downstream_ids if lib_repo.get_element_type_from_oval_id(oval_id) in args['element_type'] } message('info','Found {0} downstream OVAL ids ({1} only)'.format(len(downstream_ids), 's, '.join(args['element_type']) + 's')) print('\n{0}\n'.format(', '.join(downstream_ids))) if args['upstream']: # add all upstream element ids message('info','Finding upstream OVAL ids for {0} element(s)'.format(num_source_ids)) upstream_ids = elements_index.find_upstream_ids(source_ids, set(), distance) message('info','Found {0} upstream OVAL ids (all element types)'.format(len(upstream_ids))) if args['element_type']: upstream_ids = { oval_id for oval_id in upstream_ids if lib_repo.get_element_type_from_oval_id(oval_id) in args['element_type'] } message('info','Found {0} upstream OVAL ids ({1} only)'.format(len(upstream_ids), 's, '.join(args['element_type']) + 's')) print('\n{0}\n'.format(', '.join(upstream_ids))) seconds_elapsed = time.time() - start_time message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
def build_comprehensive_oval_document(changes): """ Builds an XML tree which contains all elements affected by the changes """ global debug global verbose if changes is None or len(changes) < 1: return None if verbose: print(" ---- Getting OVAL ID's for all changed files...") oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in changes } # find all upstream ids if verbose: print( " ---- Locating parent definitions for all changed elements...") elements_index = lib_search.ElementsIndex(False) upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set()) # filter affected to definition ids affected_def_ids = { oval_id for oval_id in upstream_ids if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition' } # get all downstream elements if verbose: print( " ---- Resolving all elements needed to build comprehensive document..." ) oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) if verbose: print( " ---- Importing separate elements into comprehensive document...." ) oval = OvalDocument(None) for path in file_paths: element = OvalElement.fromStandaloneFile(path) if element is None: print(":::: None from path: ", path) return None oval.addElement(element, True) return etree.fromstring(oval.to_string())
def build_comprehensive_oval_document(changes): """ Builds an XML tree which contains all elements affected by the changes """ global debug global verbose if changes is None or len(changes) < 1: return None if verbose: print(" ---- Getting OVAL ID's for all changed files...") oval_ids_changed = {lib_repo.path_to_oval_id(filepath) for filepath in changes} # find all upstream ids if verbose: print(" ---- Locating parent definitions for all changed elements...") elements_index = lib_search.ElementsIndex(False) upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set()) # filter affected to definition ids affected_def_ids = { oval_id for oval_id in upstream_ids if lib_repo.get_element_type_from_oval_id(oval_id) == "definition" } # get all downstream elements if verbose: print(" ---- Resolving all elements needed to build comprehensive document...") oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) if verbose: print(" ---- Importing separate elements into comprehensive document....") oval = OvalDocument(None) for path in file_paths: element = OvalElement.fromStandaloneFile(path) if element is None: print(":::: None from path: ", path) return None oval.addElement(element, True) return etree.fromstring(oval.to_string())
def get_element_change_status(left, right): """Compare two OVAL elements recursively and determine if any substantive changes have been made. @type left: Element @param left: the new element @type right: Element @param right: the existing element in the repository @rtype: int @return: And integer value indicating the level of change: -2 = an error has occurred -1 = these are not the same elements 0 = no changes or the elements are not the same item 1 = has changes, but they are inconsequential (have no affect on functionality) 2 = changes of consequence """ import traceback ignore_attributes = ['version'] unimportant_attributes = ['comment'] if left is None: if right is None: return -1 else: return False elif right is None: return -1 try: # Compare tags if left.tag != right.tag: return -1 # If this is the top-level element, make certain we really are dealing with the same elements left_id = left.get('id') if left_id is not None: # Only primary elements have OVAL IDs left_type = lib_repo.get_element_type_from_oval_id(left_id) if left_type is not None and left_type in lib_repo.supported_element_types: right_id = right.get('id') right_type = lib_repo.get_element_type_from_oval_id(right_id) if not right_id or right_id is None: return -1 if left_type != right_type: return -1 # Compare text if left.text is not None: if right.text is not None: if left.text.strip() != right.text.strip(): return 2 else: return 2 elif right.text is not None: return 2 # Compare tails? Do OVAL Elements even have tails? # if left.tail != right.tail: # return 2 # Compare attributes # But don't compare the version attribute -- so we can't just use cmp() try: for key in left.keys(): if not key in ignore_attributes: lvalue = left.get(key) rvalue = right.get(key) if lvalue != rvalue: if key in unimportant_attributes: return 1 else: return 2 except: # An exception at this point probably indicates that the attributes don't match return 2 # Compare children (recurse on this method) # Be sure we are enumerating the same children in the same order result = 0 left_children = list(left) right_children = list(right) if len(left_children) != len(right_children): return 2 for index, lchild in enumerate(left_children): # Get the matching right_child, if any # and compare children rchild = right_children[index] sub_result = get_element_change_status(lchild, rchild) if sub_result != 0 and sub_result != 1: # Anything else is reason to stop processing return sub_result elif sub_result == 1: result = 1 return result except: print (traceback.format_exc()) return -2
def main(): """ parse command line options and call lib functions """ start_time = time.time() parser = argparse.ArgumentParser(description='Identify changes in current working directory as compared to a remote authoritative copy of the repo and identify all the elements affected by those changes.') # Note: I don't think we need to support files. If a file is submitted, CIS/QA can decompose it and then run this. So this can always run against repo. #parser.add_argument('-f', '--file', required=False, help='The name of the source file. If not used the local git repository will be used as the source') parser.add_argument('--silent', required=False, action="store_true", help='Suppress messages') parser.add_argument('--remote', required=False, default='upstream', help="name of authoritative remote (default: 'upstream')") parser.add_argument('--branch', required=False, default='master', help="name of branch in authoritative remote (default: 'master')") parser.add_argument('--outfile', required=False, default='all.affected.oval.xml', help="file name OVAL definitions file containing all affected definitions (default 'all.affected.oval.xml')") parser.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the affected definitions') args = vars(parser.parse_args()) silent = args['silent'] ## 1. Find Affected Elements # get changes in working dir vs. remote/branch message('info', 'Comparing working directory to {0}/{1}'.format(args['remote'], args['branch']), silent) paths_changed = lib_git.compare_current_oval_to_remote(args['remote'], args['branch']) if not paths_changed: message('info', 'No changes. Aborting.', silent) sys.exit(0) message('info', 'Found {0} files changed in working directory:\n\t{1}'.format(len(paths_changed), '\n\t'.join(paths_changed)), silent) # convert paths to oval ids oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in paths_changed } message('info', 'Found {0} OVAL elements changed in working directory:\n\t{1}'.format(len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent) # find all upstream ids message('info','Finding upstream OVAL ids for {0} element(s)'.format(len(oval_ids_changed)), silent) elements_index = lib_search.ElementsIndex(message) upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set()) message('info','Found {0} upstream OVAL ids (all element types)'.format(len(upstream_ids)), silent) affected_oval_ids = oval_ids_changed.union(upstream_ids) # filter affected to defintion ids affected_def_ids = { oval_id for oval_id in affected_oval_ids if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition' } message('info','Found {0} upstream OVAL definitions:\n\t{1}'.format(len(affected_def_ids), '\n\t'.join(affected_def_ids)), silent) ## 2. Build an OVAL Definitions File and Validate It! message('info','Building an OVAL definitions file for all affected definitions.', silent) # get all downstream elements oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # add each OVAL definition to generator and write to file message('info',"Generating OVAL definition file '{0}' with {1} elements".format(args['outfile'], len(oval_ids)), silent) OvalGenerator = lib_xml.OvalGenerator(message) for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) OvalGenerator.to_file(args['outfile']) # validate schema_path = lib_repo.get_oval_def_schema('5.11.1') message('info','Performing schema validation', silent) try: lib_xml.schema_validate(args['outfile'], schema_path) message('info','Schema validation successful', silent) except lib_xml.SchemaValidationError as e: message('error','Schema validation failed:\n\t{0}'.format(e.message), silent) if args['schematron']: # schematron validate schema_path = lib_repo.get_oval_def_schema('5.11.1') message('info','Performing schematron validation', silent) try: lib_xml.schematron_validate(args['outfile'], schema_path) message('info','Schematron validation successful', silent) except lib_xml.SchematronValidationError as e: message('error','Schematron validation failed:\n\t{0}'.format('\n\t'.join(e.messages)), silent) #Find all downstream children -- that is, a search depth of one #Find all upstream users, all the way up to the definition #Sort the list: definitions, then tests, objects, states, and variables #Show the list #Offer to build an OVAL file that contains all the changes seconds_elapsed = time.time() - start_time message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)), silent)
def main(): start_time = time.time() tracking = dict() elements_index = lib_search.ElementsIndex(message) # getting all definitions all_def_ids = set() message('info', 'getting all definitions') for defpath in lib_repo.get_definition_paths_iterator(): all_def_ids.add(lib_repo.path_to_oval_id(defpath)) message('info', 'found {0} definitions'.format(len(all_def_ids))) # getting all element ids downstream from any definition message('info', 'getting all downstream element ids') all_downstream_ids = elements_index.find_downstream_ids(all_def_ids) message('info', 'found {0} downstream element ids'.format(len(all_downstream_ids))) # get elements that aren't in all_downstream_ids message('info', 'checking all elements') cur_element_type = None for elempath in lib_repo.get_element_paths_iterator(): oval_id = lib_repo.path_to_oval_id(elempath) element_type = lib_repo.get_element_type_from_oval_id(oval_id) # skip definitions... we're only pruning child elements if element_type == 'definition': continue # write status msg if element_type != cur_element_type: cur_element_type = element_type i_element_type = 0 i_element_type = i_element_type + 1 sys.stdout.write( 'Analyzing {0}s: {1} \r'.format( cur_element_type, i_element_type)) sys.stdout.flush() # it's an orphan if it's not downsteam of a definition track_as = 'orphan' if oval_id not in all_downstream_ids else 'in_use' if not track_as in tracking: tracking[track_as] = dict() if not element_type in tracking[track_as]: tracking[track_as][element_type] = set() tracking[track_as][element_type].add(oval_id) sys.stdout.write( ' \r'. format(cur_element_type, i_element_type)) sys.stdout.flush() # generate report report = [] for track_as, elements_by_type in tracking.items(): report.append('\t{0}:'.format(track_as.replace('_', ' ').capitalize())) for element_type, oval_ids in elements_by_type.items(): report.append('\t\t{0} {1}s'.format(len(oval_ids), element_type)) message('found', '\n'.join(report)) response = input("\n :::: Remove all orphans? (N[o] / y[es]): ") if response.lower() == 'y': orphan_ids = set() for element_type, oval_ids in tracking['orphan'].items(): orphan_ids.update(oval_ids) file_paths = elements_index.get_paths_from_ids(orphan_ids) for file_path in file_paths: message("INFO", "Deleting Orphan '%s'" % os.path.basename(file_path)) os.remove(file_path) seconds_elapsed = time.time() - start_time message('info', 'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main(): start_time = time.time() tracking = dict() elements_index = lib_search.ElementsIndex(message) # getting all definitions all_def_ids = set() message('info', 'getting all definitions') for defpath in lib_repo.get_definition_paths_iterator(): all_def_ids.add(lib_repo.path_to_oval_id(defpath)) message('info', 'found {0} definitions'.format(len(all_def_ids))) # getting all element ids downstream from any definition message('info', 'getting all downstream element ids') all_downstream_ids = elements_index.find_downstream_ids(all_def_ids) message('info', 'found {0} downstream element ids'.format(len(all_downstream_ids))) # get elements that aren't in all_downstream_ids message('info', 'checking all elements') cur_element_type = None for elempath in lib_repo.get_element_paths_iterator(): oval_id = lib_repo.path_to_oval_id(elempath) element_type = lib_repo.get_element_type_from_oval_id(oval_id) # skip definitions... we're only pruning child elements if element_type == 'definition': continue # write status msg if element_type != cur_element_type: cur_element_type = element_type i_element_type = 0 i_element_type = i_element_type + 1 sys.stdout.write('Analyzing {0}s: {1} \r'.format(cur_element_type, i_element_type)) sys.stdout.flush() # it's an orphan if it's not downsteam of a definition track_as = 'orphan' if oval_id not in all_downstream_ids else 'in_use' if not track_as in tracking: tracking[track_as] = dict() if not element_type in tracking[track_as]: tracking[track_as][element_type] = set() tracking[track_as][element_type].add(oval_id) sys.stdout.write(' \r'.format(cur_element_type, i_element_type)) sys.stdout.flush() # generate report report = [] for track_as, elements_by_type in tracking.items(): report.append('\t{0}:'.format(track_as.replace('_', ' ').capitalize())) for element_type, oval_ids in elements_by_type.items(): report.append('\t\t{0} {1}s'.format(len(oval_ids), element_type)) message('found', '\n'.join(report)) response = input("\n :::: Remove all orphans? (N[o] / y[es]): ") if response.lower() == 'y': orphan_ids = set() for element_type, oval_ids in tracking['orphan'].items(): orphan_ids.update(oval_ids) file_paths = elements_index.get_paths_from_ids(orphan_ids) for file_path in file_paths: message("INFO", "Deleting Orphan '%s'" % os.path.basename(file_path)) os.remove(file_path) seconds_elapsed = time.time() - start_time message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
# convert paths to oval ids oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in paths_changed } message('info', 'Found {0} OVAL elements changed in working directory:\n\t{1}'.format(len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent) # find all upstream ids message('info','Finding upstream OVAL ids for {0} element(s)'.format(len(oval_ids_changed)), silent) elements_index = lib_search.ElementsIndex(message) upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set()) message('info','Found {0} upstream OVAL ids (all element types)'.format(len(upstream_ids)), silent) affected_oval_ids = oval_ids_changed.union(upstream_ids) <<<<<<< HEAD # filter affected to definition ids affected_def_ids = { oval_id for oval_id in upstream_ids if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition' } message('info','Found {0} upstream OVAL definitions:\n\t{1}'.format(len(affected_def_ids), '\n\t'.join(affected_def_ids)), verbose) ======= # filter affected to defintion ids affected_def_ids = { oval_id for oval_id in affected_oval_ids if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition' } message('info','Found {0} upstream OVAL definitions:\n\t{1}'.format(len(affected_def_ids), '\n\t'.join(affected_def_ids)), silent) >>>>>>> upstream/master ## 2. Build an OVAL Definitions File and Validate It! message('info','Building an OVAL definitions file for all affected definitions.', silent) # get all downstream elements oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # add each OVAL definition to generator and write to file
def main(): """ parse command line options and call lib functions """ start_time = time.time() parser = argparse.ArgumentParser( description= 'Identify changes in current working directory as compared to a remote authoritative copy of the repo and identify all the elements affected by those changes.' ) # Note: I don't think we need to support files. If a file is submitted, CIS/QA can decompose it and then run this. So this can always run against repo. #parser.add_argument('-f', '--file', required=False, help='The name of the source file. If not used the local git repository will be used as the source') parser.add_argument('--silent', required=False, action="store_true", help='Suppress messages') parser.add_argument( '--remote', required=False, default='upstream', help="name of authoritative remote (default: 'upstream')") parser.add_argument( '--branch', required=False, default='master', help="name of branch in authoritative remote (default: 'master')") parser.add_argument( '--outfile', required=False, default='all.affected.oval.xml', help= "file name OVAL definitions file containing all affected definitions (default 'all.affected.oval.xml')" ) parser.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the affected definitions') args = vars(parser.parse_args()) silent = args['silent'] ## 1. Find Affected Elements # get changes in working dir vs. remote/branch message( 'info', 'Comparing working directory to {0}/{1}'.format( args['remote'], args['branch']), silent) paths_changed = lib_git.compare_current_oval_to_remote( args['remote'], args['branch']) if not paths_changed: message('info', 'No changes. Aborting.', silent) sys.exit(0) message( 'info', 'Found {0} files changed in working directory:\n\t{1}'.format( len(paths_changed), '\n\t'.join(paths_changed)), silent) # convert paths to oval ids oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in paths_changed } message( 'info', 'Found {0} OVAL elements changed in working directory:\n\t{1}'.format( len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent) # find all upstream ids message( 'info', 'Finding upstream OVAL ids for {0} element(s)'.format( len(oval_ids_changed)), silent) elements_index = lib_search.ElementsIndex(message) upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set()) message( 'info', 'Found {0} upstream OVAL ids (all element types)'.format( len(upstream_ids)), silent) affected_oval_ids = oval_ids_changed.union(upstream_ids) # filter affected to definition ids affected_def_ids = { oval_id for oval_id in upstream_ids if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition' } message( 'info', 'Found {0} upstream OVAL definitions:\n\t{1}'.format( len(affected_def_ids), '\n\t'.join(affected_def_ids)), silent) ## 2. Build an OVAL Definitions File and Validate It! message('info', 'Building an OVAL definitions file for all affected definitions.', silent) # get all downstream elements oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids) file_paths = elements_index.get_paths_from_ids(oval_ids) # add each OVAL definition to generator and write to file message( 'info', "Generating OVAL definition file '{0}' with {1} elements".format( args['outfile'], len(oval_ids)), silent) OvalGenerator = lib_xml.OvalGenerator(message) for file_path in file_paths: element_type = lib_repo.get_element_type_from_path(file_path) OvalGenerator.queue_element_file(element_type, file_path) OvalGenerator.to_file(args['outfile']) # validate schema_path = lib_repo.get_oval_def_schema('5.11.1') message('info', 'Performing schema validation', silent) try: lib_xml.schema_validate(args['outfile'], schema_path) message('info', 'Schema validation successful', silent) except lib_xml.SchemaValidationError as e: message('error', 'Schema validation failed:\n\t{0}'.format(e.message), silent) if args['schematron']: # schematron validate schema_path = lib_repo.get_oval_def_schema('5.11.1') message('info', 'Performing schematron validation', silent) try: lib_xml.schematron_validate(args['outfile'], schema_path) message('info', 'Schematron validation successful', silent) except lib_xml.SchematronValidationError as e: message( 'error', 'Schematron validation failed:\n\t{0}'.format( '\n\t'.join(e.messages)), silent) #Find all downstream children -- that is, a search depth of one #Find all upstream users, all the way up to the definition #Sort the list: definitions, then tests, objects, states, and variables #Show the list #Offer to build an OVAL file that contains all the changes seconds_elapsed = time.time() - start_time message('info', 'Completed in {0}!'.format(format_duration(seconds_elapsed)), silent)