Example #1
0
def determine_def_min_version(defpath, definitions_index, elements_index, update):
    """ determines the minimum oval schema version the given definition can be expressed in. """

    global schema_path_cache

    # build schema path cache
    for schema_version in lib_repo.get_schema_versions():
        schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)

    minimum_version=None
    exception=None

    # get id of oval definition
    def_id = lib_repo.path_to_oval_id(defpath)

    # add all downstream element ids
    def_ids = set([def_id])
    oval_ids = elements_index.find_downstream_ids(def_ids, def_ids)
    file_paths = elements_index.get_paths_from_ids(oval_ids)

    # create generator that builds files in memory (these are small)
    OvalGenerator = lib_xml.OvalGenerator(message)
    OvalGenerator.use_file_queues = False

    # add each OVAL definition to generator
    for file_path in file_paths:
        element_type = lib_repo.get_element_type_from_path(file_path)
        OvalGenerator.queue_element_file(element_type, file_path)

    # parse defintion, get ref to schema_version element
    xmlstring = OvalGenerator.to_string(True, False)
    xmlstring = xmlstring.replace('<?xml version="1.0" encoding="UTF-8"?>','')
    xmlstring = xmlstring.replace('<?xml version="1.0" encoding="utf-8"?>','')
    xmlstring = xmlstring.replace("<?xml version='1.0' encoding='UTF-8'?>",'')
    xmlstring = xmlstring.replace("<?xml version='1.0' encoding='utf-8'?>",'')
    #message("INFO", "XMLSTRING is %s" % xmlstring)
    #tree = etree.fromstring(OvalGenerator.to_string(True, False))
    tree = etree.XML(xmlstring)
    schema_version_element = tree.find('.//oval:schema_version', { 'oval': 'http://oval.mitre.org/XMLSchema/oval-common-5' })

    for schema_version in lib_repo.get_schema_versions():
        # update schema version in tree
        schema_version_element.text = schema_version

        # test of definitions file validates against current schema
        try:
            message("INFO", "Schema Path is %s" % schema_path_cache[schema_version])
            lib_xml.schema_validate(tree, schema_path_cache[schema_version])
            minimum_version = schema_version
            message("INFO", "XML is valid against schema version %s" % schema_version)
        except lib_xml.SchemaValidationError as e:
            exception = e.message
            message("ERROR", "Exception (schema version is %s) - %s" % (schema_version, e.message))
            break;

    if update:
        set_min_schema_version(defpath, minimum_version)

    return { 'oval_id': def_id, 'minimum_version': minimum_version, 'exception': exception }
Example #2
0
def main():
    """ parse command line options and generate file """
    start_time = time.time()

    # parse command line options
    parser = argparse.ArgumentParser(
        description='Schema validates one or more OVAL definitions files.')
    parser.add_argument(
        'pathname',
        nargs='+',
        help=
        'pathname(s) specifying file(s) to validate (supports glob patterns)')
    parser.add_argument(
        '-s',
        '--schematron',
        default=False,
        action="store_true",
        help='additionally, schematron validate the output file')
    args = vars(parser.parse_args())

    # find and normalize all files in provided glob(s)
    files = {
        os.path.normpath(pathname)
        for pathname in args['pathname'] for filepath in glob.glob(pathname)
    }

    files_count = len(files)
    message('info', 'Preparing to validate {0} file(s)'.format(files_count))

    schema_path = lib_repo.get_oval_def_schema('5.11.2')
    for i_file, filepath in enumerate(files):
        message('info', '{0}. {1}'.format(i_file + 1, filepath))

        # schema validate
        message('info', 'Performing schema validation')
        try:
            lib_xml.schema_validate(filepath, schema_path)
            message('info', 'Schema validation successful')
        except lib_xml.SchemaValidationError as e:
            message('error',
                    'Schema validation failed:\n\t{0}'.format(e.message))

        if args['schematron']:
            # schematron validate
            message('info', 'Performing schematron validation')
            try:
                lib_xml.schematron_validate(filepath, schema_path)
                message('info', 'Schematron validation successful')
            except lib_xml.SchematronValidationError as e:
                message(
                    'error', 'Schematron validation failed:\n\t{0}'.format(
                        '\n\t'.join(e.messages)))

    seconds_elapsed = time.time() - start_time
    message('info',
            'Completed in {0}!'.format(format_duration(seconds_elapsed)))
Example #3
0
def main():
    """
    Breaks the OVAL file into its constituent elements and writes each of those into the repository
    """

    global verbose
    global force
    global schema_path

    parser = argparse.ArgumentParser(
        description=
        'Separates a file into its component parts and saves them to the repository.'
    )
    program_options = parser.add_argument_group('options')
    program_options.add_argument('files',
                                 metavar='<FILE>',
                                 nargs='+',
                                 help='The file or files to process')
    program_options.add_argument(
        '-f',
        '--force',
        required=False,
        action="store_true",
        help=
        'Ignore the check for unchanged items and write all elements to the repository'
    )
    program_options.add_argument('-v',
                                 '--verbose',
                                 required=False,
                                 action="store_true",
                                 help='Enable more verbose messages')

    args = parser.parse_args()
    options = vars(args)

    if options['verbose']:
        verbose = True
    else:
        verbose = False

    if options['force']:
        force = True
    else:
        force = False

    schema_path = lib_repo.get_oval_def_schema(SCHEMA_VERSION)

    files = args.files
    if files is None or len(files) < 1:
        print("\n ## Error:  must include at least one file name")
        return

    for file in files:
        if not submit_contents(file):
            print(
                "\n### Error in file '{0}' -- processing halted".format(file))
def determine_def_min_version(defpath, definitions_index, elements_index, update):
    """ determines the minimum oval schema version the given definition can be expressed in. """

    global schema_path_cache

    # build schema path cache
    for schema_version in lib_repo.get_schema_versions():
        schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)

    minimum_version=None
    exception=None

    # get id of oval definition
    def_id = lib_repo.path_to_oval_id(defpath)

    # add all downstream element ids
    def_ids = set([def_id])
    oval_ids = elements_index.find_downstream_ids(def_ids, def_ids)
    file_paths = elements_index.get_paths_from_ids(oval_ids)

    # create generator that builds files in memory (these are small)
    OvalGenerator = lib_xml.OvalGenerator(message)
    OvalGenerator.use_file_queues = False

    # add each OVAL definition to generator
    for file_path in file_paths:
        element_type = lib_repo.get_element_type_from_path(file_path)
        OvalGenerator.queue_element_file(element_type, file_path)

    # parse defintion, get ref to schema_version element
    tree = etree.fromstring(OvalGenerator.to_string(True, False))
    schema_version_element = tree.find('.//oval:schema_version', { 'oval': 'http://oval.mitre.org/XMLSchema/oval-common-5' })

    for schema_version in lib_repo.get_schema_versions():
        # update schema version in tree
        schema_version_element.text = schema_version

        # test of definitions file validates against current schema
        try:
            message("INFO", "Schema Path is %s" % schema_path_cache[schema_version])
            lib_xml.schema_validate(tree, schema_path_cache[schema_version])
            minimum_version = schema_version
            message("INFO", "XML is valid against schema version %s" % schema_version)
        except lib_xml.SchemaValidationError as e:
            exception = e.message
            message("ERROR", "Exception (schema version is %s) - %s" % (schema_version, e.message))
            break;

    if update:
        set_min_schema_version(defpath, minimum_version)

    return { 'oval_id': def_id, 'minimum_version': minimum_version, 'exception': exception }
def main():
    """
    Breaks the OVAL file into its constituent elements and writes each of those into the repository
    """

    global verbose
    global force
    global schema_path

    parser = argparse.ArgumentParser(
        description="Separates a file into its component parts and saves them to the repository."
    )
    program_options = parser.add_argument_group("options")
    program_options.add_argument("files", metavar="<FILE>", nargs="+", help="The file or files to process")
    program_options.add_argument(
        "-f",
        "--force",
        required=False,
        action="store_true",
        help="Ignore the check for unchanged items and write all elements to the repository",
    )
    program_options.add_argument(
        "-v", "--verbose", required=False, action="store_true", help="Enable more verbose messages"
    )

    args = parser.parse_args()
    options = vars(args)

    if options["verbose"]:
        verbose = True
    else:
        verbose = False

    if options["force"]:
        force = True
    else:
        force = False

    schema_path = lib_repo.get_oval_def_schema(SCHEMA_VERSION)

    files = args.files
    if files is None or len(files) < 1:
        print("\n ## Error:  must include at least one file name")
        return

    for file in files:
        if not submit_contents(file):
            print("\n### Error in file '{0}' -- processing halted".format(file))
Example #6
0
def schema_validate(oval_tree, schema_version):
        
    global verbose
    
    schema_path = lib_repo.get_oval_def_schema(schema_version)
    try:
        lib_xml.schema_validate(oval_tree, schema_path)
    except Exception as e:
        print('    Schema validation failed:\n\t{0}'.format(e.message))
        return False

    # schematron validate
    print('info','Performing schematron validation')
    try:
        lib_xml.schematron_validate_tree(oval_tree, schema_path)
    except Exception as e:
        print('    Schematron validation failed:\n\t{0}'.format('\n\t'.join(e.messages)))
        return False;
        
    return True
Example #7
0
def schema_validate(oval_tree, schema_version):
        
    global verbose
    
    schema_path = lib_repo.get_oval_def_schema(schema_version)
    try:
        lib_xml.schema_validate(oval_tree, schema_path)
    except Exception as e:
        print('    Schema validation failed:\n\t{0}'.format(e.message))
        return False

    # schematron validate
    print('info','Performing schematron validation')
    try:
        lib_xml.schematron_validate_tree(oval_tree, schema_path)
    except Exception as e:
        print('    Schematron validation failed:\n\t{0}'.format('\n\t'.join(e.messages)))
        return False;
        
    return True
def main():
    """ parse command line options and generate file """
    start_time = time.time()

    # parse command line options
    parser = argparse.ArgumentParser(description='Schema validates one or more OVAL definitions files.')
    parser.add_argument('pathname', nargs='+', help='pathname(s) specifying file(s) to validate (supports glob patterns)')
    parser.add_argument('-s', '--schematron', default=False, action="store_true", help='additionally, schematron validate the output file')
    args = vars(parser.parse_args())

    # find and normalize all files in provided glob(s)
    files = { os.path.normpath(pathname) for pathname in args['pathname'] for filepath in glob.glob(pathname) }

    files_count = len(files)
    message('info', 'Preparing to validate {0} file(s)'.format(files_count))
    
    schema_path = lib_repo.get_oval_def_schema('5.11.1') 
    for i_file, filepath in enumerate(files):
        message('info','{0}. {1}'.format(i_file + 1, filepath))

        # schema validate
        message('info','Performing schema validation')
        try:
            lib_xml.schema_validate(filepath, schema_path)
            message('info','Schema validation successful')
        except lib_xml.SchemaValidationError as e:
            message('error','Schema validation failed:\n\t{0}'.format(e.message))

        if args['schematron']:
            # schematron validate
            message('info','Performing schematron validation')
            try:
                lib_xml.schematron_validate(filepath, schema_path)
                message('info','Schematron validation successful')
            except lib_xml.SchematronValidationError as e:
                message('error','Schematron validation failed:\n\t{0}'.format('\n\t'.join(e.messages)))

    seconds_elapsed = time.time() - start_time
    message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
Example #9
0
def main():
    
    SCHEMA_VERSION = '5.11.1'

    global verbose
    global debug
    global autoaccept
    
    parser = argparse.ArgumentParser(description='Performs all of the identified QA tasks against an OVAL submission in the repository')
    output_options = parser.add_argument_group('verbosity options')
    output_options.add_argument('-v', '--verbose', default=False, action="store_true", help='Verbose progress messages')
    output_options.add_argument('-d', '--debug', default=False, action="store_true", help='Include debug information on errors')
#     output_options.add_argument('-a', '--autoaccept', default=False, action="store_true", help='Automatically continue without prompting when it is safe to do so')

    args = vars(parser.parse_args())

    if args['verbose']:
        verbose = True
        
    if args['debug']:
        debug = True
        
#     if args['autoaccept']:
#         autoaccept = True
    
    # Grab some things we're going to need later
    # First, build the schema path cache
    element_index = lib_search.ElementsIndex(message)
    schema_path_cache = {}
    for schema_version in lib_repo.get_schema_versions():
        schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)

    
    # 1. Locate all uncommitted changes to the local repository
    if verbose:
        print("\n + 1:  looking for uncommitted changes")
    change_list = lib_git.get_uncommitted_oval()
    if change_list is None or len(change_list) < 1:
        print("\n-------- This update does not include any changes of significance --------\n")
        return


    # 1.1 Determine which of these changes are due to removed files    
    if verbose:
        print("   +++ Number of changed items to QA: {0}".format(len(change_list)))
    remove_list = find_removed_items(change_list)
    
    # 1.2 Issue warning (prompt to continue) if any of the changes are a deleted item
    if verbose:
        print("  + 1.2: Determining if any changes are deleted items...")
    if remove_list is not None and len(remove_list) > 0:
        print("\n  -------- The following files were removed as a part of this update --------\n")
        show_files(remove_list)
        # TODO:  Offer the option to inspect the OVALIDs in the removed files and 
        #    build a list of what items, if any, refer to them
        response = input("\n :: Accept these changes (N[o] / y[es] / s[how affected]): ")
        if response == 's' or response == 'S':
            show_affected(remove_list)
            response = input("\n :::: Accept these changes (N[o] / y[es]): ")
            if response != 'y' and response != 'Y':
                return
        elif response != 'y' and response != 'Y':
            return
        # 1.3 Don't include removed files as part of the update
        change_list = [file for file in change_list if file not in remove_list]
        
        
    elif verbose:
        print("    +++ No removed items found")
        
    

    # 2. Remove all changes that are semantically the same as existing elements (except for states)
    if verbose:
        print("\n + 2: Removing items that don't contain meaningful changes...")
    change_list = prune_unchanged_elements(change_list)
    
    # 2.1 If that means we have no changes left, there is nothing else to do
    if len(change_list) < 1:
        print("\n  ----- This update does not include any changes of significance")
        return
    print("\n  ---- Number of changed elements to process: {0}\n".format(len(change_list)))
    
    
    # 3. For each element in the list that is a definition, check:
    if verbose:
        print(" + 3: Checking correctness of definition metadata")
        

    def_list = [ path for path in change_list if lib_repo.get_element_type_from_path(path) == 'definition']
    if def_list is not None and len(def_list) > 0:
        valid_metadata = 1
        if verbose:
            print("   +++ Number of definitions in this update: {0}".format(len(def_list)))
        for def_path in def_list:
            def_element = lib_xml.load_standalone_element(def_path)

            ode = lib_oval.OvalElement(def_element)
            od  = lib_oval.OvalDefinition(ode.getElement())

            def_id = od.getId()

        # 3.1 If this is an update, does it change any existing metadata?
        # 3.2 Check existence and accuracy of definition metadata (<status> and date)
        #  - INITIAL SUBMISSION or DRAFT on new submission
        #  - INTERIM if updating a previous definition
        #  - ?

            # no <dates> - invalid
            # @version == 0:
            #   no <submitted> - invalid
            #   <status_change>s > 0 - invalid
            #   <status> != "INITIAL SUBMISSION" - invalid
            # @ version > 0:
            #   last <status_change> != <status> - invalid
            def_status_change = od.get_last_status_change()
            if def_status_change["Version"] == "0":
                if "Submitted" not in def_status_change or def_status_change["Submitted"] is None:
                    print("   ++++ Definition ID %s is NOT valid:" % def_id)
                    print("    - New definitions must contain a submitted element")
                    valid_metadata = 0

                if def_status_change["StatusChange"]:
                    print("   ++++ Definition ID %s is NOT valid:" % def_id)
                    print("    - New definitions should not contain a status change element")
                    valid_metadata = 0

                if def_status_change["Status"] != "INITIAL SUBMISSION":
                    print("   ++++ Definition ID %s is NOT valid:" % def_id)
                    print("    - New definitions must have a status of INITIAL SUBMISSION")
                    valid_metadata = 0
            else:
                defstatus = def_status_change["Status"]
                lscstatus = def_status_change["StatusChange"]["Status"]
                if (defstatus != lscstatus):
                    print("   ++++ Definition ID %s is NOT valid:" % def_id)
                    print("    - Last status change (%s) does not match definition status (%s)" % (lscstatus, defstatus))
                    valid_metadata = 0

        if valid_metadata == 0:
            print("\n   ++++ Definition Metadata is Invalid.  Exiting...")
            return

    elif verbose:
        print("   +++ No definitions to check")
    
    
    # 4. Schema validate the changes
    # First, generate an OVAL document
    if verbose:
        print("\n + 4: Schema validating changes...")
        
    schema_path = lib_repo.get_oval_def_schema(SCHEMA_VERSION)
    for element_file in change_list:
        try:
            lib_xml.schema_validate(element_file, schema_path, True)
        except Exception as e:
            print('    Schema validation failed:\n\t{0}'.format(e.message))
            print("\n ### Offending file {0}".format(element_file))
            return

    if verbose:
        print(" ++++ Schema validations passed")
        print("\n + 5: Updating elements...")    

    # 5. On passing all of the above, make these changes for all elements:


    oval_id_map = {}
    affected_elements = set()
    update_elements = {}
    for path in change_list:
        oval_element = lib_xml.load_standalone_element(path)
        update_elements[path] = oval_element
        #  5.1 If it's a definition, determine and set the minimum schema version
        ovalid = oval_element.get("id")
        if verbose:
            print("\n  ---- Processing submitted element {0}".format(ovalid))
            
        if lib_repo.get_element_type_from_path(path) == 'definition':
            if verbose:
                print("    --- Is a definition:  determining minimum schema version")
#             min_schema = determine_definition_minimum_schema_version(path, element_index, schema_path_cache)
            min_schema = determine_definition_minimum_schema_version(oval_element, element_index, schema_path_cache)
            if min_schema and min_schema is not None:
                if verbose:
                    print("    ---- Determined minimum schema version to be {0}".format(min_schema))
                set_minimum_schema_version(oval_element, min_schema)
        
        #  5.2 For each element that is not using an OVALID in the CIS namespace:
        is_update = True
        if not is_repository_id(ovalid):
            is_update = False
            element_type = lib_repo.get_element_type_from_path(path)
            new_id = generate_next_ovalid(element_type, element_index)
            if verbose:
                print("    ---- Change submission ID from '{0}' to '{1}'".format(ovalid, new_id))
            oval_element.set("id", new_id)
            #      5.2.1 Set to a unique OVALID in the CIS namespace
            #      5.2.2 Update all references from the old OVALID
            oval_id_map[ovalid] = new_id
        
        #  5.3 Set/update version numbers as necessary.  The previous step can be used to determine new vice update
        if is_update:
        #    5.3.1 If this is an update, find the current version and increment by one
            if verbose:
                print("     ---- Is an update:  incrementing version")
            increment_version(oval_element)
            # Find all upstream elements and add them, as unique, to the list of items to change
            if lib_repo.get_element_type_from_path(path) != 'definition':
                if verbose:
                    print("     ---- Not a definition.  Finding affected upstream elements...")
                affected = find_affected(ovalid, element_index)
                if affected is not None and len(affected) > 0:
                    if verbose:
                        print("     ---- Number of elements affected: {0}".format(len(affected)))
                    affected_elements = set().union(affected_elements, affected)
                else:
                    if verbose:
                        print("     >>>>> Warning: found no affected elements for this update.  Possible orphan.")
        else:
        #        Otherwise, set it to 1
            oval_element.set("version", "1")
        #  5.4 Canonicalize all altered elements (if possible)
        
    # Now that we know all the elements affected by an update we can increment their IDs once
    if len(affected_elements) > 0:
        if verbose:
            print("\n ------- This update affects {0} upstream elements:  incrementing the version number for each...".format(len(affected_elements)))
        for file in affected_elements:
            oval_element = lib_xml.load_standalone_element(file)
            if oval_element is not None:
                increment_version(oval_element)
                #oval_element = normalize_ids(oval_element, oval_id_map)
                update_elements[file] = oval_element
    
    
    #  6 Write the element, and remove the old if the path changed
    print("\n=============== Complete ===================")
    print("All automated checks have completed successfully, but the following")
    print(" manual checks need to be made prior to accepting this submission:")
    print("   * Metadata for definitions is complete and accurate")
    print("   * Existing metadata has not been changed")
    print("   * Contains a meaningful change")
    print("   * Does not contain any harmful actions or unacceptable language")

    for x in oval_id_map:
        print(" -- Convert %s to %s" % (x, oval_id_map[x]))

    response = input("\n :::: Save all changes now? (N[o] / y[es]): ")
    if response != 'y' and response != 'Y':
        return

    for path in update_elements:
        oval_element = normalize_ids(update_elements[path], oval_id_map)
        if not oval_element or oval_element is None:
            continue
        new_path = lib_repo.get_element_repository_path(oval_element)
        if verbose:
            print("## Writing {0}".format(new_path))
        save_element(oval_element, new_path)
        
        if new_path != path:
            if verbose:
                print("### Deleting {0}".format(path))
            try:
                os.remove(path)
            except Exception:
                print("#### Exception/Skipping Deleting {0}".format(path))
            
    
    # 7. Prompt for a message to use for the commit
    # 7.1 Commit and push the changes
    
    return
Example #10
0
def main():
    """ parse command line options and call lib functions """
    start_time = time.time()

    parser = argparse.ArgumentParser(description='Identify changes in current working directory as compared to a remote authoritative copy of the repo and identify all the elements affected by those changes.')
    # Note: I don't think we need to support files. If a file is submitted, CIS/QA can decompose it and then run this. So this can always run against repo.
    #parser.add_argument('-f', '--file', required=False, help='The name of the source file.  If not used the local git repository will be used as the source')
    parser.add_argument('--silent', required=False, action="store_true", help='Suppress messages')
    parser.add_argument('--remote', required=False, default='upstream', help="name of authoritative remote (default: 'upstream')")
    parser.add_argument('--branch', required=False, default='master', help="name of branch in authoritative remote (default: 'master')")
    parser.add_argument('--outfile', required=False, default='all.affected.oval.xml', help="file name OVAL definitions file containing all affected definitions (default 'all.affected.oval.xml')")
    parser.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the affected definitions')
    args = vars(parser.parse_args())

    silent = args['silent']

    ## 1. Find Affected Elements

    # get changes in working dir vs. remote/branch
    message('info', 'Comparing working directory to {0}/{1}'.format(args['remote'], args['branch']), silent)
    paths_changed = lib_git.compare_current_oval_to_remote(args['remote'], args['branch'])
    if not paths_changed:
        message('info', 'No changes. Aborting.', silent)
        sys.exit(0)
    message('info', 'Found {0} files changed in working directory:\n\t{1}'.format(len(paths_changed), '\n\t'.join(paths_changed)), silent)

    # convert paths to oval ids
    oval_ids_changed = { lib_repo.path_to_oval_id(filepath) for filepath in paths_changed }
    message('info', 'Found {0} OVAL elements changed in working directory:\n\t{1}'.format(len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent)

    # find all upstream ids
    message('info','Finding upstream OVAL ids for {0} element(s)'.format(len(oval_ids_changed)), silent)
    elements_index = lib_search.ElementsIndex(message)
    upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set())
    message('info','Found {0} upstream OVAL ids (all element types)'.format(len(upstream_ids)), silent)

    affected_oval_ids = oval_ids_changed.union(upstream_ids)

    # filter affected to defintion ids
    affected_def_ids = { oval_id for oval_id in affected_oval_ids if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition' }
    message('info','Found {0} upstream OVAL definitions:\n\t{1}'.format(len(affected_def_ids), '\n\t'.join(affected_def_ids)), silent)

    ## 2. Build an OVAL Definitions File and Validate It!
    message('info','Building an OVAL definitions file for all affected definitions.', silent)

    # get all downstream elements
    oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids)
    file_paths = elements_index.get_paths_from_ids(oval_ids)

    # add each OVAL definition to generator and write to file
    message('info',"Generating OVAL definition file '{0}' with {1} elements".format(args['outfile'], len(oval_ids)), silent)
    OvalGenerator = lib_xml.OvalGenerator(message)
    for file_path in file_paths:
        element_type = lib_repo.get_element_type_from_path(file_path)
        OvalGenerator.queue_element_file(element_type, file_path)
    OvalGenerator.to_file(args['outfile'])

    # validate
    schema_path = lib_repo.get_oval_def_schema('5.11.1')
    message('info','Performing schema validation', silent)
    try:
        lib_xml.schema_validate(args['outfile'], schema_path)
        message('info','Schema validation successful', silent)
    except lib_xml.SchemaValidationError as e:
        message('error','Schema validation failed:\n\t{0}'.format(e.message), silent)

    if args['schematron']:
        # schematron validate
        schema_path = lib_repo.get_oval_def_schema('5.11.1')
        message('info','Performing schematron validation', silent)
        try:
            lib_xml.schematron_validate(args['outfile'], schema_path)
            message('info','Schematron validation successful', silent)
        except lib_xml.SchematronValidationError as e:
            message('error','Schematron validation failed:\n\t{0}'.format('\n\t'.join(e.messages)), silent)

    #Find all downstream children -- that is, a search depth of one
    #Find all upstream users, all the way up to the definition
    
    #Sort the list:  definitions, then tests, objects, states, and variables
    #Show the list
    #Offer to build an OVAL file that contains all the changes

    seconds_elapsed = time.time() - start_time
    message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)), silent)
Example #11
0
def main():
    
    SCHEMA_VERSION = '5.11.1'

    global verbose
    global debug
    global autoaccept
    
    parser = argparse.ArgumentParser(description='Performs all of the identified QA tasks against an OVAL submission in the repository')
    output_options = parser.add_argument_group('verbosity options')
    output_options.add_argument('-v', '--verbose', default=False, action="store_true", help='Verbose progress messages')
    output_options.add_argument('-d', '--debug', default=False, action="store_true", help='Include debug information on errors')
#     output_options.add_argument('-a', '--autoaccept', default=False, action="store_true", help='Automatically continue without prompting when it is safe to do so')

    args = vars(parser.parse_args())

    if args['verbose']:
        verbose = True
        
    if args['debug']:
        debug = True
        
#     if args['autoaccept']:
#         autoaccept = True
    
    # Grab some things we're going to need later
    # First, build the schema path cache
    element_index = lib_search.ElementsIndex(message)
    schema_path_cache = {}
    for schema_version in lib_repo.get_schema_versions():
        schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)

    
    # 1. Locate all uncommitted changes to the local repository
    if verbose:
        print("\n + 1:  looking for uncommitted changes")
    change_list = lib_git.get_uncommitted_oval()
    if change_list is None or len(change_list) < 1:
        print("\n-------- This update does not include any changes of significance --------\n")
        return


    # 1.1 Determine which of these changes are due to removed files    
    if verbose:
        print("   +++ Number of changed items to QA: {0}".format(len(change_list)))
    remove_list = find_removed_items(change_list)
    
    # 1.2 Issue warning (prompt to continue) if any of the changes are a deleted item
    if verbose:
        print("  + 1.2: Determining if any changes are deleted items...")
    if remove_list is not None and len(remove_list) > 0:
        print("\n  -------- The following files were removed as a part of this update --------\n")
        show_files(remove_list)
        # TODO:  Offer the option to inspect the OVALIDs in the removed files and 
        #    build a list of what items, if any, refer to them
        response = input("\n :: Accept these changes (N[o] / y[es] / s[how affected]): ")
        if response == 's' or response == 'S':
            show_affected(remove_list)
            response = input("\n :::: Accept these changes (N[o] / y[es]): ")
            if response != 'y' and response != 'Y':
                return
        elif response != 'y' and response != 'Y':
            return
        # 1.3 Don't include removed files as part of the update
        change_list = [file for file in change_list if file not in remove_list]
        
        
    elif verbose:
        print("    +++ No removed items found")
        
    

    # 2. Remove all changes that are semantically the same as existing elements (except for states)
    if verbose:
        print("\n + 2: Removing items that don't contain meaningful changes...")
    change_list = prune_unchanged_elements(change_list)
    
    # 2.1 If that means we have no changes left, there is nothing else to do
    if len(change_list) < 1:
        print("\n  ----- This update does not include any changes of significance")
        return
    print("\n  ---- Number of changed elements to process: {0}\n".format(len(change_list)))
    
    
    # 3. For each element in the list that is a definition, check:
    if verbose:
        print(" + 3: Checking correctness of definition metadata")
        

    def_list = [ path for path in change_list if lib_repo.get_element_type_from_path(path) == 'definition']
    if def_list is not None and len(def_list) > 0:
        valid_metadata = 1
        if verbose:
            print("   +++ Number of definitions in this update: {0}".format(len(def_list)))
        for def_path in def_list:
            def_element = lib_xml.load_standalone_element(def_path)

            ode = lib_oval.OvalElement(def_element)
            od  = lib_oval.OvalDefinition(ode.getElement())

            def_id = od.getId()

        # 3.1 If this is an update, does it change any existing metadata?
        # 3.2 Check existence and accuracy of definition metadata (<status> and date)
        #  - INITIAL SUBMISSION or DRAFT on new submission
        #  - INTERIM if updating a previous definition
        #  - ?

            # no <dates> - invalid
            # @version == 0:
            #   no <submitted> - invalid
            #   <status_change>s > 0 - invalid
            #   <status> != "INITIAL SUBMISSION" - invalid
            # @ version > 0:
            #   last <status_change> != <status> - invalid
            def_status_change = od.get_last_status_change()
            if def_status_change["Version"] == "0":
                if "Submitted" not in def_status_change or def_status_change["Submitted"] is None:
                    print("   ++++ Definition ID %s is NOT valid:" % def_id)
                    print("    - New definitions must contain a submitted element")
                    valid_metadata = 0

                if def_status_change["StatusChange"]:
                    print("   ++++ Definition ID %s is NOT valid:" % def_id)
                    print("    - New definitions should not contain a status change element")
                    valid_metadata = 0

                if def_status_change["Status"] != "INITIAL SUBMISSION":
                    print("   ++++ Definition ID %s is NOT valid:" % def_id)
                    print("    - New definitions must have a status of INITIAL SUBMISSION")
                    valid_metadata = 0
            else:
                defstatus = def_status_change["Status"]
                lscstatus = def_status_change["StatusChange"]["Status"]
                if (defstatus != lscstatus):
                    print("   ++++ Definition ID %s is NOT valid:" % def_id)
                    print("    - Last status change (%s) does not match definition status (%s)" % (lscstatus, defstatus))
                    valid_metadata = 0

        if valid_metadata == 0:
            print("\n   ++++ Definition Metadata is Invalid.  Exiting...")
            return

    elif verbose:
        print("   +++ No definitions to check")
    
    
    # 4. Schema validate the changes
    # First, generate an OVAL document
    if verbose:
        print("\n + 4: Schema validating changes...")
        
    schema_path = lib_repo.get_oval_def_schema(SCHEMA_VERSION)
    for element_file in change_list:
        try:
            lib_xml.schema_validate(element_file, schema_path, True)
        except Exception as e:
            print('    Schema validation failed:\n\t{0}'.format(e.message))
            print("\n ### Offending file {0}".format(element_file))
            return

    if verbose:
        print(" ++++ Schema validations passed")
        print("\n + 5: Updating elements...")    

    # 5. On passing all of the above, make these changes for all elements:


    oval_id_map = {}
    affected_elements = set()
    update_elements = {}
    for path in change_list:
        oval_element = lib_xml.load_standalone_element(path)
        update_elements[path] = oval_element
        #  5.1 If it's a definition, determine and set the minimum schema version
        ovalid = oval_element.get("id")
        if verbose:
            print("\n  ---- Processing submitted element {0}".format(ovalid))
            
        if lib_repo.get_element_type_from_path(path) == 'definition':
            if verbose:
                print("    --- Is a definition:  determining minimum schema version")
#             min_schema = determine_definition_minimum_schema_version(path, element_index, schema_path_cache)
            min_schema = determine_definition_minimum_schema_version(oval_element, element_index, schema_path_cache)
            if min_schema and min_schema is not None:
                if verbose:
                    print("    ---- Determined minimum schema version to be {0}".format(min_schema))
                set_minimum_schema_version(oval_element, min_schema)
        
        #  5.2 For each element that is not using an OVALID in the CIS namespace:
        is_update = True
        if not is_repository_id(ovalid):
            is_update = False
            element_type = lib_repo.get_element_type_from_path(path)
            new_id = generate_next_ovalid(element_type, element_index)
            if verbose:
                print("    ---- Change submission ID from '{0}' to '{1}'".format(ovalid, new_id))
            oval_element.set("id", new_id)
            #      5.2.1 Set to a unique OVALID in the CIS namespace
            #      5.2.2 Update all references from the old OVALID
            oval_id_map[ovalid] = new_id
        
        #  5.3 Set/update version numbers as necessary.  The previous step can be used to determine new vice update
        if is_update:
        #    5.3.1 If this is an update, find the current version and increment by one
            if verbose:
                print("     ---- Is an update:  incrementing version")
            increment_version(oval_element)
            # Find all upstream elements and add them, as unique, to the list of items to change
            if lib_repo.get_element_type_from_path(path) != 'definition':
                if verbose:
                    print("     ---- Not a definition.  Finding affected upstream elements...")
                affected = find_affected(ovalid, element_index)
                if affected is not None and len(affected) > 0:
                    if verbose:
                        print("     ---- Number of elements affected: {0}".format(len(affected)))
                    affected_elements = set().union(affected_elements, affected)
                else:
                    if verbose:
                        print("     >>>>> Warning: found no affected elements for this update.  Possible orphan.")
        else:
        #        Otherwise, set it to 1
            oval_element.set("version", "1")
        #  5.4 Canonicalize all altered elements (if possible)
        
    # Now that we know all the elements affected by an update we can increment their IDs once
    if len(affected_elements) > 0:
        if verbose:
            print("\n ------- This update affects {0} upstream elements:  incrementing the version number for each...".format(len(affected_elements)))
        for file in affected_elements:
            oval_element = lib_xml.load_standalone_element(file)
            if oval_element is not None:
                increment_version(oval_element)
                #oval_element = normalize_ids(oval_element, oval_id_map)
                update_elements[file] = oval_element
    
    
    #  6 Write the element, and remove the old if the path changed
    print("\n=============== Complete ===================")
    print("All automated checks have completed successfully, but the following")
    print(" manual checks need to be made prior to accepting this submission:")
    print("   * Metadata for definitions is complete and accurate")
    print("   * Existing metadata has not been changed")
    print("   * Contains a meaningful change")
    print("   * Does not contain any harmful actions or unacceptable language")

    for x in oval_id_map:
        print(" -- Convert %s to %s" % (x, oval_id_map[x]))

    response = input("\n :::: Save all changes now? (N[o] / y[es]): ")
    if response != 'y' and response != 'Y':
        return

    for path in update_elements:
        oval_element = normalize_ids(update_elements[path], oval_id_map)
        if not oval_element or oval_element is None:
            continue
        new_path = lib_repo.get_element_repository_path(oval_element)
        if verbose:
            print("## Writing {0}".format(new_path))
        save_element(oval_element, new_path)
        
        if new_path != path:
            if verbose:
                print("### Deleting {0}".format(path))
            os.remove(path)
            
    
    # 7. Prompt for a message to use for the commit
    # 7.1 Commit and push the changes
    
    return
def main():
    """ parse command line options and generate file """
    start_time = time.time()

    # parse command line options
    parser = argparse.ArgumentParser(description='Builds a schema-valid OVAL definitions file.')
    output_options = parser.add_argument_group('output options')
    output_options.add_argument('-o', '--outfile', required=True, help='file name for output OVAL definitions file')
    output_options.add_argument('-v', '--validate', default=False, action="store_true", help='schema validate the output file')
    output_options.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the output file')
    source_options = parser.add_argument_group('definitions filtering',
                                               'Provide at least one of the following options to determine which definition(s) ' +
                                               'will be included. Results will include the intersection of matches for each parameter ' +
                                               'supplied. When multiple values are supplied for one paramater, the parameter will ' +
                                               'match definitions that match any provided value.')
    source_options.add_argument('--definition_id', nargs='*', dest='oval_id', help='match OVAL definition id(s)')
    source_options.add_argument('--title', nargs='*', dest='title', metavar='PHRASE', help='match phrase(s) in definition titles')
    source_options.add_argument('--description', nargs='*', dest='description', metavar='PHRASE', help='match phrase(s) in definition titles')
    source_options.add_argument('--class', nargs='*', dest='class', help='filter by class(es): {0}'.format(', '.join(lib_repo.supported_definition_classes)))
    source_options.add_argument('--status', nargs='*', dest='status', help='filter by status(es): {0}'.format(', '.join(lib_repo.supported_definition_statuses)))
    source_options.add_argument('--family', nargs='*', dest='family', help='filter by family(ies)')
    source_options.add_argument('--platform', nargs='*', dest='platforms', metavar='PLATFORM', help='filter by platform(s)')
    source_options.add_argument('--product', nargs='*', dest='products', metavar='PRODUCT', help='filter by product(s)')
    source_options.add_argument('--contributor', nargs='*', dest='contributors', metavar='NAME', help='filter by contributor(s)')
    source_options.add_argument('--organization', nargs='*', dest='organizations', metavar='NAME', help='filter by organization(s)')
    source_options.add_argument('--reference_id', nargs='*', dest='reference_ids', metavar='REFERENCE_ID', help='filter by reference ids, e.g. CVE-2015-3306')
    output_options.add_argument('--all_definitions', default=False, action="store_true", help='include all definitions in the repository (do not specify any other filters)')
    args = vars(parser.parse_args())

    # get definitions index
    definitions_index = lib_search.DefinitionsIndex(message)

    # contruct query from args
    query = {}
    for field in definitions_index.get_fieldnames():
        if field in args and args[field]:
            query[field] = args[field]

    # --all_definitions OR at least one definition selection option must be specified
    if args['all_definitions'] and query:
        parser.print_help()
        message('error',"The '--all_definitions' filter cannot be combined with any other filters.")
        sys.exit(0)
    elif not (args['all_definitions'] or query):
        parser.print_help()
        message('error','At least one definitions filtering argument must be provided.')
        sys.exit(0)

    # query index
    query_results = definitions_index.query(query)
    if not query_results:
        message('info','No matching OVAL definitions found. Aborting.')
        sys.exit(0)

    # get set of all definition ids found
    definition_ids = { document['oval_id'] for document in query_results }
    message('info','Found {0} matching OVAL definitions'.format(len(definition_ids)))

    # add all downstream element ids
    message('info','Finding downstream OVAL ids for all definitions')
    elements_index = lib_search.ElementsIndex(message)
    oval_ids = elements_index.find_downstream_ids(definition_ids, definition_ids)
    message('info','Found {0} downstream OVAL ids'.format(len(oval_ids) - len(query_results)))

    # get paths for all elements
    message('info','Finding paths for {0} OVAL elements'.format(len(oval_ids)))
    file_paths = elements_index.get_paths_from_ids(oval_ids)

    # create generator
    OvalGenerator = lib_xml.OvalGenerator(message)

    # add each OVAL definition to generator
    message('info','generating OVAL definition file with {0} elements'.format(len(oval_ids)))
    for file_path in file_paths:
        element_type = lib_repo.get_element_type_from_path(file_path)
        OvalGenerator.queue_element_file(element_type, file_path)

    # write output file
    message('info','writing OVAL definitions to {0}'.format(args['outfile']))
    OvalGenerator.write(args['outfile'])

    # validate
    if args['validate']:
        # schema validate
        schema_path = lib_repo.get_oval_def_schema('5.11.1')
        message('info','performing schema validation')
        try:
            lib_xml.schema_validate(args['outfile'], schema_path)
            message('info','schema validation successful')
        except lib_xml.SchemaValidationError as e:
            message('error','schema validation failed:\n\t{0}'.format(e.message))

    if args['schematron']:
        # schematron validate
        schema_path = lib_repo.get_oval_def_schema('5.11.1')
        message('info','performing schematron validation')
        try:
            lib_xml.schematron_validate(args['outfile'], schema_path)
            message('info','schematron validation successful')
        except lib_xml.SchematronValidationError as e:
            message('error','schematron validation failed:\n\t{0}'.format('\n\t'.join(e.messages)))

    seconds_elapsed = time.time() - start_time
    message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
Example #13
0
    message('info','Building an OVAL definitions file for all affected definitions.', silent)

    # get all downstream elements
    oval_ids = elements_index.find_downstream_ids(affected_def_ids, affected_def_ids)
    file_paths = elements_index.get_paths_from_ids(oval_ids)

    # add each OVAL definition to generator and write to file
    message('info',"Generating OVAL definition file '{0}' with {1} elements".format(args['outfile'], len(oval_ids)), silent)
    OvalGenerator = lib_xml.OvalGenerator(message)
    for file_path in file_paths:
        element_type = lib_repo.get_element_type_from_path(file_path)
        OvalGenerator.queue_element_file(element_type, file_path)
    OvalGenerator.to_file(args['outfile'])

    # validate
    schema_path = lib_repo.get_oval_def_schema('5.11.1')
    message('info','Performing schema validation', silent)
    try:
        lib_xml.schema_validate(args['outfile'], schema_path)
        message('info','Schema validation successful', silent)
    except lib_xml.SchemaValidationError as e:
        message('error','Schema validation failed:\n\t{0}'.format(e.message), silent)

    if args['schematron']:
        # schematron validate
        schema_path = lib_repo.get_oval_def_schema('5.11.1')
        message('info','Performing schematron validation', silent)
        try:
            lib_xml.schematron_validate(args['outfile'], schema_path)
            message('info','Schematron validation successful', silent)
        except lib_xml.SchematronValidationError as e:
Example #14
0
def main():
    """ parse command line options and call lib functions """
    start_time = time.time()

    parser = argparse.ArgumentParser(
        description=
        'Identify changes in current working directory as compared to a remote authoritative copy of the repo and identify all the elements affected by those changes.'
    )
    # Note: I don't think we need to support files. If a file is submitted, CIS/QA can decompose it and then run this. So this can always run against repo.
    #parser.add_argument('-f', '--file', required=False, help='The name of the source file.  If not used the local git repository will be used as the source')
    parser.add_argument('--silent',
                        required=False,
                        action="store_true",
                        help='Suppress messages')
    parser.add_argument(
        '--remote',
        required=False,
        default='upstream',
        help="name of authoritative remote (default: 'upstream')")
    parser.add_argument(
        '--branch',
        required=False,
        default='master',
        help="name of branch in authoritative remote (default: 'master')")
    parser.add_argument(
        '--outfile',
        required=False,
        default='all.affected.oval.xml',
        help=
        "file name OVAL definitions file containing all affected definitions (default 'all.affected.oval.xml')"
    )
    parser.add_argument('-s',
                        '--schematron',
                        default=False,
                        action="store_true",
                        help='schematron validate the affected definitions')
    args = vars(parser.parse_args())

    silent = args['silent']

    ## 1. Find Affected Elements

    # get changes in working dir vs. remote/branch
    message(
        'info', 'Comparing working directory to {0}/{1}'.format(
            args['remote'], args['branch']), silent)
    paths_changed = lib_git.compare_current_oval_to_remote(
        args['remote'], args['branch'])
    if not paths_changed:
        message('info', 'No changes. Aborting.', silent)
        sys.exit(0)
    message(
        'info', 'Found {0} files changed in working directory:\n\t{1}'.format(
            len(paths_changed), '\n\t'.join(paths_changed)), silent)

    # convert paths to oval ids
    oval_ids_changed = {
        lib_repo.path_to_oval_id(filepath)
        for filepath in paths_changed
    }
    message(
        'info',
        'Found {0} OVAL elements changed in working directory:\n\t{1}'.format(
            len(oval_ids_changed), '\n\t'.join(oval_ids_changed)), silent)

    # find all upstream ids
    message(
        'info', 'Finding upstream OVAL ids for {0} element(s)'.format(
            len(oval_ids_changed)), silent)
    elements_index = lib_search.ElementsIndex(message)
    upstream_ids = elements_index.find_upstream_ids(oval_ids_changed, set())
    message(
        'info', 'Found {0} upstream OVAL ids (all element types)'.format(
            len(upstream_ids)), silent)

    affected_oval_ids = oval_ids_changed.union(upstream_ids)

    # filter affected to definition ids
    affected_def_ids = {
        oval_id
        for oval_id in upstream_ids
        if lib_repo.get_element_type_from_oval_id(oval_id) == 'definition'
    }
    message(
        'info', 'Found {0} upstream OVAL definitions:\n\t{1}'.format(
            len(affected_def_ids), '\n\t'.join(affected_def_ids)), silent)

    ## 2. Build an OVAL Definitions File and Validate It!
    message('info',
            'Building an OVAL definitions file for all affected definitions.',
            silent)

    # get all downstream elements
    oval_ids = elements_index.find_downstream_ids(affected_def_ids,
                                                  affected_def_ids)
    file_paths = elements_index.get_paths_from_ids(oval_ids)

    # add each OVAL definition to generator and write to file
    message(
        'info',
        "Generating OVAL definition file '{0}' with {1} elements".format(
            args['outfile'], len(oval_ids)), silent)
    OvalGenerator = lib_xml.OvalGenerator(message)
    for file_path in file_paths:
        element_type = lib_repo.get_element_type_from_path(file_path)
        OvalGenerator.queue_element_file(element_type, file_path)
    OvalGenerator.to_file(args['outfile'])

    # validate
    schema_path = lib_repo.get_oval_def_schema('5.11.1')
    message('info', 'Performing schema validation', silent)
    try:
        lib_xml.schema_validate(args['outfile'], schema_path)
        message('info', 'Schema validation successful', silent)
    except lib_xml.SchemaValidationError as e:
        message('error', 'Schema validation failed:\n\t{0}'.format(e.message),
                silent)

    if args['schematron']:
        # schematron validate
        schema_path = lib_repo.get_oval_def_schema('5.11.1')
        message('info', 'Performing schematron validation', silent)
        try:
            lib_xml.schematron_validate(args['outfile'], schema_path)
            message('info', 'Schematron validation successful', silent)
        except lib_xml.SchematronValidationError as e:
            message(
                'error', 'Schematron validation failed:\n\t{0}'.format(
                    '\n\t'.join(e.messages)), silent)

    #Find all downstream children -- that is, a search depth of one
    #Find all upstream users, all the way up to the definition

    #Sort the list:  definitions, then tests, objects, states, and variables
    #Show the list
    #Offer to build an OVAL file that contains all the changes

    seconds_elapsed = time.time() - start_time
    message('info',
            'Completed in {0}!'.format(format_duration(seconds_elapsed)),
            silent)
def main():
    """ parse command line options and generate file """
    start_time = time.time()

    # parse command line options
    parser = argparse.ArgumentParser(
        description='Builds a schema-valid OVAL definitions file.')
    output_options = parser.add_argument_group('output options')
    output_options.add_argument(
        '-o',
        '--outfile',
        required=True,
        help='file name for output OVAL definitions file')
    output_options.add_argument('-v',
                                '--validate',
                                default=False,
                                action="store_true",
                                help='schema validate the output file')
    output_options.add_argument('-s',
                                '--schematron',
                                default=False,
                                action="store_true",
                                help='schematron validate the output file')
    output_options.add_argument(
        '-t',
        '--tempdir',
        required=False,
        default="./",
        help=
        "directory to store temporary files used when building oval definitions (default: './')"
    )
    source_options = parser.add_argument_group(
        'definitions filtering',
        'Provide at least one of the following options to determine which definition(s) '
        +
        'will be included. Results will include the intersection of matches for each parameter '
        +
        'supplied. When multiple values are supplied for one paramater, the parameter will '
        + 'match definitions that match any provided value.')
    source_options.add_argument('--definition_id',
                                nargs='*',
                                dest='oval_id',
                                help='match OVAL definition id(s)')
    source_options.add_argument('--title',
                                nargs='*',
                                dest='title',
                                metavar='PHRASE',
                                help='match phrase(s) in definition titles')
    source_options.add_argument('--description',
                                nargs='*',
                                dest='description',
                                metavar='PHRASE',
                                help='match phrase(s) in definition titles')
    source_options.add_argument(
        '--class',
        nargs='*',
        dest='class',
        help='filter by class(es): {0}'.format(', '.join(
            lib_repo.supported_definition_classes)))
    source_options.add_argument(
        '--status',
        nargs='*',
        dest='status',
        help='filter by status(es): {0}'.format(', '.join(
            lib_repo.supported_definition_statuses)))
    source_options.add_argument('--family',
                                nargs='*',
                                dest='family',
                                help='filter by family(ies)')
    source_options.add_argument('--platform',
                                nargs='*',
                                dest='platforms',
                                metavar='PLATFORM',
                                help='filter by platform(s)')
    source_options.add_argument('--product',
                                nargs='*',
                                dest='products',
                                metavar='PRODUCT',
                                help='filter by product(s)')
    source_options.add_argument('--contributor',
                                nargs='*',
                                dest='contributors',
                                metavar='NAME',
                                help='filter by contributor(s)')
    source_options.add_argument('--organization',
                                nargs='*',
                                dest='organizations',
                                metavar='NAME',
                                help='filter by organization(s)')
    source_options.add_argument(
        '--reference_id',
        nargs='*',
        dest='reference_ids',
        metavar='REFERENCE_ID',
        help='filter by reference ids, e.g. CVE-2015-3306')
    source_options.add_argument(
        '--max_schema_version',
        nargs="?",
        dest='max_schema_version',
        metavar='SCHEMA_VERSION',
        help='filter by maximum oval schema version, e.g. 5.10')
    source_options.add_argument(
        '--all_definitions',
        default=False,
        action="store_true",
        help=
        'include all definitions in the repository (do not specify any other filters)'
    )
    source_options.add_argument(
        '--from',
        nargs='?',
        default='',
        metavar='YYYYMMDD',
        help='include elements revised on or after this day (format: YYYYMMDD)'
    )
    source_options.add_argument(
        '--to',
        nargs='?',
        default='',
        metavar='YYYYMMDD',
        help='include elements revised on or before this day (format: YYYYMMDD)'
    )
    args = vars(parser.parse_args())

    # get definitions index
    definitions_index = lib_search.DefinitionsIndex(message)

    # contruct query from args
    query = {}
    for field in definitions_index.get_fieldnames():
        if field in args and args[field]:
            query[field] = args[field]

    # add schema_version filter, if specified
    if args['max_schema_version']:
        query['min_schema_version'] = '[0 TO {0}]'.format(
            definitions_index.version_to_int(args['max_schema_version']))

    # add date range and contributor/org filters, if specified
    all_definitions_filtered = False
    if args['from'] or args['to'] or args['contributors'] or args[
            'organizations']:
        # get revisions index
        revisions_index = lib_search.RevisionsIndex(message)

        if args['from'] or args['to']:
            filtered_oval_ids = revisions_index.get_definition_ids({
                'date':
                revisions_index.format_daterange(args['from'], args['to'])
            })

        if args['contributors']:
            contributor_filtered_ids = revisions_index.get_definition_ids(
                {'contributor': args['contributors']})
            filtered_oval_ids = filtered_oval_ids & contributor_filtered_ids if 'filtered_oval_ids' in locals(
            ) else contributor_filtered_ids

        if args['organizations']:
            organization_filtered_ids = revisions_index.get_definition_ids(
                {'organization': args['organizations']})
            filtered_oval_ids = filtered_oval_ids & organization_filtered_ids if 'filtered_oval_ids' in locals(
            ) else organization_filtered_ids

        # add to query
        if 'oval_id' in query and query['oval_id']:
            # if oval_id(s) specified in args, get intersection with filtered oval ids
            query['oval_id'] = set(query['oval_id']) & filtered_oval_ids
        else:
            query['oval_id'] = filtered_oval_ids

        if not query['oval_id']:
            all_definitions_filtered = True

    # --all_definitions OR at least one definition selection option must be specified
    if args['all_definitions'] and query:
        parser.print_help()
        message(
            'error',
            "The '--all_definitions' filter cannot be combined with any other filters."
        )
        sys.exit(1)
    elif not (args['all_definitions'] or query):
        parser.print_help()
        message(
            'error',
            'At least one definitions filtering argument must be provided.')
        sys.exit(1)

    # query index
    query_results = definitions_index.query(
        query) if not all_definitions_filtered else {}

    # get set of all definition ids found
    definition_ids = {document['oval_id'] for document in query_results}
    message('info',
            'Found {0} matching OVAL definitions'.format(len(definition_ids)))

    # create generator and set oval schema version, if necessary
    OvalGenerator = lib_xml.OvalGenerator(message, args['tempdir'])
    if args['max_schema_version']:
        OvalGenerator.oval_schema_version = args['max_schema_version']

    if definition_ids:
        # add all downstream element ids
        message('info', 'Finding downstream OVAL ids for all definitions')
        elements_index = lib_search.ElementsIndex(message)
        oval_ids = elements_index.find_downstream_ids(definition_ids,
                                                      definition_ids)
        message(
            'info', 'Found {0} downstream OVAL ids'.format(
                len(oval_ids) - len(query_results)))

        # get paths for all elements
        message('info',
                'Finding paths for {0} OVAL elements'.format(len(oval_ids)))
        file_paths = elements_index.get_paths_from_ids(oval_ids)

        # build in memory if there aren't that many files
        if len(file_paths) < 200:
            OvalGenerator.use_file_queues = False

        # add each OVAL definition to generator
        message(
            'info', 'Generating OVAL definition file with {0} elements'.format(
                len(oval_ids)))
        for file_path in file_paths:
            element_type = lib_repo.get_element_type_from_path(file_path)
            OvalGenerator.queue_element_file(element_type, file_path)

    # write output file
    message('info', 'Writing OVAL definitions to {0}'.format(args['outfile']))
    OvalGenerator.to_file(args['outfile'])

    # validate
    if args['validate']:
        # schema validate
        schema_path = lib_repo.get_oval_def_schema(
            OvalGenerator.oval_schema_version)
        message('info', 'performing schema validation')
        try:
            lib_xml.schema_validate(args['outfile'], schema_path)
            message('info', 'schema validation successful')
        except lib_xml.SchemaValidationError as e:
            message('error',
                    'schema validation failed:\n\t{0}'.format(e.message))

    if args['schematron']:
        # schematron validate
        schema_path = lib_repo.get_oval_def_schema(
            OvalGenerator.oval_schema_version)
        message('info', 'performing schematron validation')
        try:
            lib_xml.schematron_validate(args['outfile'], schema_path)
            message('info', 'schematron validation successful')
        except lib_xml.SchematronValidationError as e:
            message(
                'error', 'schematron validation failed:\n\t{0}'.format(
                    '\n\t'.join(e.messages)))

    seconds_elapsed = time.time() - start_time
    message('info',
            'Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main():
    """ parse command line options and generate file """
    start_time = time.time()

    # parse command line options
    parser = argparse.ArgumentParser(description='Builds a schema-valid OVAL definitions file.')
    output_options = parser.add_argument_group('output options')
    output_options.add_argument('-o', '--outfile', required=True, help='file name for output OVAL definitions file')
    output_options.add_argument('-v', '--validate', default=False, action="store_true", help='schema validate the output file')
    output_options.add_argument('-s', '--schematron', default=False, action="store_true", help='schematron validate the output file')
    output_options.add_argument('-t', '--tempdir', required=False, default="./", help="directory to store temporary files used when building oval definitions (default: './')")
    source_options = parser.add_argument_group('definitions filtering',
                                               'Provide at least one of the following options to determine which definition(s) ' +
                                               'will be included. Results will include the intersection of matches for each parameter ' +
                                               'supplied. When multiple values are supplied for one paramater, the parameter will ' +
                                               'match definitions that match any provided value.')
    source_options.add_argument('--definition_id', nargs='*', dest='oval_id', help='match OVAL definition id(s)')
    source_options.add_argument('--title', nargs='*', dest='title', metavar='PHRASE', help='match phrase(s) in definition titles')
    source_options.add_argument('--description', nargs='*', dest='description', metavar='PHRASE', help='match phrase(s) in definition titles')
    source_options.add_argument('--class', nargs='*', dest='class', help='filter by class(es): {0}'.format(', '.join(lib_repo.supported_definition_classes)))
    source_options.add_argument('--status', nargs='*', dest='status', help='filter by status(es): {0}'.format(', '.join(lib_repo.supported_definition_statuses)))
    source_options.add_argument('--family', nargs='*', dest='family', help='filter by family(ies)')
    source_options.add_argument('--platform', nargs='*', dest='platforms', metavar='PLATFORM', help='filter by platform(s)')
    source_options.add_argument('--product', nargs='*', dest='products', metavar='PRODUCT', help='filter by product(s)')
    source_options.add_argument('--contributor', nargs='*', dest='contributors', metavar='NAME', help='filter by contributor(s)')
    source_options.add_argument('--organization', nargs='*', dest='organizations', metavar='NAME', help='filter by organization(s)')
    source_options.add_argument('--reference_id', nargs='*', dest='reference_ids', metavar='REFERENCE_ID', help='filter by reference ids, e.g. CVE-2015-3306')
    source_options.add_argument('--max_schema_version', nargs="?", dest='max_schema_version', metavar='SCHEMA_VERSION',  help='filter by maximum oval schema version, e.g. 5.10')
    source_options.add_argument('--all_definitions', default=False, action="store_true", help='include all definitions in the repository (do not specify any other filters)')
    source_options.add_argument('--from', nargs='?', default='', metavar='YYYYMMDD', help='include elements revised on or after this day (format: YYYYMMDD)')
    source_options.add_argument('--to', nargs='?', default='', metavar='YYYYMMDD', help='include elements revised on or before this day (format: YYYYMMDD)')
    args = vars(parser.parse_args())

    # get definitions index
    definitions_index = lib_search.DefinitionsIndex(message)

    # contruct query from args
    query = {}
    for field in definitions_index.get_fieldnames():
        if field in args and args[field]:
            query[field] = args[field]

    # add schema_version filter, if specified
    if args['max_schema_version']:
        query['min_schema_version'] = '[0 TO {0}]'.format(definitions_index.version_to_int(args['max_schema_version']))

    # add date range and contributor/org filters, if specified
    all_definitions_filtered = False
    if args['from'] or args['to'] or args['contributors'] or args['organizations']:
        # get revisions index
        revisions_index = lib_search.RevisionsIndex(message)

        if args['from'] or args['to']:
            filtered_oval_ids = revisions_index.get_definition_ids({ 'date': revisions_index.format_daterange(args['from'], args['to']) })

        if args['contributors']:
            contributor_filtered_ids = revisions_index.get_definition_ids({ 'contributor': args['contributors'] })
            filtered_oval_ids = filtered_oval_ids & contributor_filtered_ids if 'filtered_oval_ids' in locals() else contributor_filtered_ids

        if args['organizations']:
            organization_filtered_ids = revisions_index.get_definition_ids({ 'organization': args['organizations'] })
            filtered_oval_ids = filtered_oval_ids & organization_filtered_ids if 'filtered_oval_ids' in locals() else organization_filtered_ids

        # add to query
        if 'oval_id' in query and query['oval_id']:
            # if oval_id(s) specified in args, get intersection with filtered oval ids
            query['oval_id'] = set(query['oval_id']) & filtered_oval_ids
        else:
            query['oval_id'] = filtered_oval_ids

        if not query['oval_id']:
            all_definitions_filtered = True

    # --all_definitions OR at least one definition selection option must be specified
    if args['all_definitions'] and query:
        parser.print_help()
        message('error',"The '--all_definitions' filter cannot be combined with any other filters.")
        sys.exit(1)
    elif not (args['all_definitions'] or query):
        parser.print_help()
        message('error','At least one definitions filtering argument must be provided.')
        sys.exit(1)

    # query index
    query_results = definitions_index.query(query) if not all_definitions_filtered else {}

    # get set of all definition ids found
    definition_ids = { document['oval_id'] for document in query_results }
    message('info','Found {0} matching OVAL definitions'.format(len(definition_ids)))

    # create generator and set oval schema version, if necessary
    OvalGenerator = lib_xml.OvalGenerator(message, args['tempdir'])
    if args['max_schema_version']:
        OvalGenerator.oval_schema_version = args['max_schema_version']

    if definition_ids:
        # add all downstream element ids
        message('info','Finding downstream OVAL ids for all definitions')
        elements_index = lib_search.ElementsIndex(message)
        oval_ids = elements_index.find_downstream_ids(definition_ids, definition_ids)
        message('info','Found {0} downstream OVAL ids'.format(len(oval_ids) - len(query_results)))

        # get paths for all elements
        message('info','Finding paths for {0} OVAL elements'.format(len(oval_ids)))
        file_paths = elements_index.get_paths_from_ids(oval_ids)

        # build in memory if there aren't that many files
        if len(file_paths) < 200:
            OvalGenerator.use_file_queues = False

        # add each OVAL definition to generator
        message('info','Generating OVAL definition file with {0} elements'.format(len(oval_ids)))
        for file_path in file_paths:
            element_type = lib_repo.get_element_type_from_path(file_path)
            OvalGenerator.queue_element_file(element_type, file_path)

    # write output file
    message('info','Writing OVAL definitions to {0}'.format(args['outfile']))
    OvalGenerator.to_file(args['outfile'])

    # validate
    if args['validate']:
        # schema validate
        schema_path = lib_repo.get_oval_def_schema(OvalGenerator.oval_schema_version)
        message('info','performing schema validation')
        try:
            lib_xml.schema_validate(args['outfile'], schema_path)
            message('info','schema validation successful')
        except lib_xml.SchemaValidationError as e:
            message('error','schema validation failed:\n\t{0}'.format(e.message))

    if args['schematron']:
        # schematron validate
        schema_path = lib_repo.get_oval_def_schema(OvalGenerator.oval_schema_version)
        message('info','performing schematron validation')
        try:
            lib_xml.schematron_validate(args['outfile'], schema_path)
            message('info','schematron validation successful')
        except lib_xml.SchematronValidationError as e:
            message('error','schematron validation failed:\n\t{0}'.format('\n\t'.join(e.messages)))

    seconds_elapsed = time.time() - start_time
    message('info','Completed in {0}!'.format(format_duration(seconds_elapsed)))
def main():
    """ parse command line options and generate file """
    start_time = time.time()

    # parse command line options
    parser = argparse.ArgumentParser(description="Determine minimum supported OVAL schema for a given definition")
    parser.add_argument("-p", "--path", help="path to definition fragment")
    parser.add_argument("-i", "--id", help="id of definition fragment")
    parser.add_argument(
        "-a",
        "--all",
        default=False,
        action="store_true",
        help="determine minimum supported OVAL schema for all indexed definitino fragments. cannot be combined with --id or --path",
    )
    parser.add_argument(
        "-u",
        "--update",
        default=False,
        action="store_true",
        help="update the given definition to include the min_schema_version element",
    )

    args = vars(parser.parse_args())

    if args["all"] and (args["id"] or args["path"]):
        parser.print_help()
        message("error", "--all cannot be combined with --id or --path.")
        sys.exit(0)

    if not args["all"] and not (args["path"] or args["id"]):
        parser.print_help()
        message("error", "Please specify one of: --all, --path or --id.")
        sys.exit(0)

    if args["path"] and args["id"]:
        parser.print_help()
        message("error", "Please specify one of: --all, --path, or --id.")
        sys.exit(0)

    # get indexes
    definitions_index = lib_search.DefinitionsIndex(message)
    elements_index = lib_search.ElementsIndex(message)

    # build schema path cache
    for schema_version in lib_repo.get_schema_versions():
        schema_path_cache[schema_version] = lib_repo.get_oval_def_schema(schema_version)

    if args["all"]:
        for defpath in lib_repo.get_definition_paths_iterator():
            result = determine_def_min_version(defpath, definitions_index, elements_index, args["update"])
            report(defpath, result["minimum_version"])

    elif args["id"]:
        documents = definitions_index.query({"oval_id": args["id"]})

        if len(documents) == 0:
            message("info", "No definitions having id '{0}' found.".format(args["id"]))
            sys.exit(0)

        paths = [document["path"] for document in documents]

        for path in paths:
            result = determine_def_min_version(path, definitions_index, elements_index, args["update"])
            report(result["oval_id"], result["minimum_version"])

    elif args["path"] and not os.path.isfile(args["path"]):
        message("info", "Definition fragment path not found: " + args["path"])
        sys.exit(0)
    else:
        result = determine_def_min_version(args["path"], definitions_index, elements_index, args["update"])
        report(args["path"], result["minimum_version"])

    seconds_elapsed = time.time() - start_time
    report("info", "Completed in {0}!".format(format_duration(seconds_elapsed)))