def find_rules(directory, func): results = [] product_yamls = {} product_yaml_paths = {} for root, dirs, files in os.walk(directory): product_yaml = None product_yaml_path = None if "product.yml" in files: product_yaml_path = os.path.join(root, "product.yml") product_yaml = ssgcommon.open_yaml(product_yaml_path) product_yamls[root] = product_yaml product_yaml_paths[root] = product_yaml_path for d in dirs: product_yamls[os.path.join(root, d)] = product_yaml product_yaml_paths[os.path.join(root, d)] = product_yaml_path elif root in product_yamls: product_yaml = product_yamls[root] product_yaml_path = product_yaml_paths[root] for d in dirs: product_yamls[os.path.join(root, d)] = product_yaml product_yaml_paths[os.path.join(root, d)] = product_yaml_path else: pass # print("No product yaml for file: %s" % root) for filename in files: path = os.path.join(root, filename) if len(path) < 5 or path[-5:] != '.rule': continue if func(path, product_yaml): results.append((path, product_yaml_path)) return results
def find_rules(directory, func): # Iterates over passed directory to correctly parse rules (which are # YAML files with internal macros). The most recently seen product.yml # takes precedence over previous product.yml, e.g.: # # a/product.yml # a/b/product.yml -- will be selected for the following rule: # a/b/c/something.rule # # The corresponding rule and contents of the product.yml are then passed # into func(/path/to/rule, product_yaml_contents); if the result evalutes # to true, the tuple (/path/to/rule, /path/to/product.yml) is saved as a # result. # # This process mimics the build system and allows us to find rule files # which satisfy the constraints of the passed func. results = [] product_yamls = {} product_yaml_paths = {} for root, dirs, files in os.walk(directory): product_yaml = None product_yaml_path = None if "product.yml" in files: product_yaml_path = os.path.join(root, "product.yml") product_yaml = ssgcommon.open_yaml(product_yaml_path) product_yamls[root] = product_yaml product_yaml_paths[root] = product_yaml_path for d in dirs: product_yamls[os.path.join(root, d)] = product_yaml product_yaml_paths[os.path.join(root, d)] = product_yaml_path elif root in product_yamls: product_yaml = product_yamls[root] product_yaml_path = product_yaml_paths[root] for d in dirs: product_yamls[os.path.join(root, d)] = product_yaml product_yaml_paths[os.path.join(root, d)] = product_yaml_path else: pass for filename in files: path = os.path.join(root, filename) if len(path) < 5 or path[-5:] != '.rule': continue try: if func(path, product_yaml): results.append((path, product_yaml_path)) except jinja2.exceptions.UndefinedError: print( "Failed to parse file %s (with product.yaml: %s). Skipping" % (path, product_yaml_path)) pass return results
def from_yaml(yaml_file, product_yaml=None): yaml_contents = open_yaml(yaml_file, product_yaml) if yaml_contents is None: return None group_id, _ = os.path.splitext(os.path.basename(yaml_file)) group = Group(group_id) group.prodtype = yaml_contents.get("prodtype", "all") group.title = required_yaml_key(yaml_contents, "title") group.description = required_yaml_key(yaml_contents, "description") return group
def from_yaml(yaml_file, product_yaml=None): yaml_contents = open_yaml(yaml_file, product_yaml) if yaml_contents is None: return None value_id, _ = os.path.splitext(os.path.basename(yaml_file)) value = Value(value_id) value.title = required_yaml_key(yaml_contents, "title") value.description = required_yaml_key(yaml_contents, "description") value.type = required_yaml_key(yaml_contents, "type") value.options = required_yaml_key(yaml_contents, "options") return value
def fix_empty_identifiers(directory): results = find_rules(directory, has_empty_identifier) print("Number of rules with empty identifiers: %d" % len(results)) for result in results: rule_path = result[0] product_yaml_path = result[1] product_yaml = None if product_yaml_path is not None: product_yaml = ssgcommon.open_yaml(product_yaml_path) fix_empty_identifier(rule_path, product_yaml)
def find_int_references(directory): results = find_rules(directory, has_int_reference) print("Number of rules with integer references: %d" % len(results)) for result in results: rule_path = result[0] product_yaml_path = result[1] product_yaml = None if product_yaml_path is not None: product_yaml = ssgcommon.open_yaml(product_yaml_path) fix_file(rule_path, product_yaml, fix_int_reference)
def from_yaml(yaml_file, product_yaml=None): yaml_contents = open_yaml(yaml_file, product_yaml) if yaml_contents is None: return None basename, _ = os.path.splitext(os.path.basename(yaml_file)) profile = Profile(basename) profile.title = required_yaml_key(yaml_contents, "title") profile.description = required_yaml_key(yaml_contents, "description") profile.extends = yaml_contents.get("extends", None) profile.selections = required_yaml_key(yaml_contents, "selections") return profile
def find_prefix_cce(directory): results = find_rules(directory, has_prefix_cce) print("Number of rules with prefixed CCEs: %d" % len(results)) for result in results: rule_path = result[0] rule_path = result[0] product_yaml_path = result[1] product_yaml = None if product_yaml_path is not None: product_yaml = ssgcommon.open_yaml(product_yaml_path) fix_file(rule_path, product_yaml, fix_prefix_cce)
def from_yaml(yaml_file, product_yaml=None): yaml_contents = open_yaml(yaml_file, product_yaml) if yaml_contents is None: return None group_id, _ = os.path.splitext(os.path.basename(yaml_file)) group = Group(group_id) group.prodtype = yaml_contents.get("prodtype", "all") group.title = required_yaml_key(yaml_contents, "title") group.description = required_yaml_key(yaml_contents, "description") group.warnings = yaml_contents.get("warnings", []) for warning_list in group.warnings: if len(warning_list) != 1: raise ValueError("Only one key/value pair should exist for each dictionary") return group
def from_yaml(yaml_file, product_yaml=None): yaml_contents = open_yaml(yaml_file, product_yaml) if yaml_contents is None: return None value_id, _ = os.path.splitext(os.path.basename(yaml_file)) value = Value(value_id) value.title = required_yaml_key(yaml_contents, "title") value.description = required_yaml_key(yaml_contents, "description") value.type = required_yaml_key(yaml_contents, "type") value.options = required_yaml_key(yaml_contents, "options") value.warnings = yaml_contents.get("warnings", []) for warning_list in value.warnings: if len(warning_list) != 1: raise ValueError("Only one key/value pair should exist for each dictionary") return value
def from_yaml(yaml_file, product_yaml=None): yaml_contents = open_yaml(yaml_file, product_yaml) if yaml_contents is None: return None rule_id, _ = os.path.splitext(os.path.basename(yaml_file)) rule = Rule(rule_id) rule.prodtype = yaml_contents.get("prodtype", "all") rule.title = required_yaml_key(yaml_contents, "title") rule.description = required_yaml_key(yaml_contents, "description") rule.rationale = required_yaml_key(yaml_contents, "rationale") rule.severity = required_yaml_key(yaml_contents, "severity") rule.references = yaml_contents.get("references", []) rule.identifiers = yaml_contents.get("identifiers", []) rule.ocil_clause = yaml_contents.get("ocil_clause") rule.ocil = yaml_contents.get("ocil") rule.external_oval = yaml_contents.get("oval_external_content") return rule
def from_yaml(yaml_file, id_, product_yaml=None): yaml_contents = open_yaml(yaml_file, product_yaml) if yaml_contents is None: return None benchmark = Benchmark(id_) benchmark.title = required_yaml_key(yaml_contents, "title") benchmark.status = required_yaml_key(yaml_contents, "status") benchmark.description = required_yaml_key(yaml_contents, "description") notice_contents = required_yaml_key(yaml_contents, "notice") benchmark.notice_id = required_yaml_key(notice_contents, "id") benchmark.notice_description = required_yaml_key( notice_contents, "description") benchmark.front_matter = required_yaml_key(yaml_contents, "front-matter") benchmark.rear_matter = required_yaml_key(yaml_contents, "rear-matter") benchmark.cpes = yaml_contents.get("cpes", []) benchmark.version = str(required_yaml_key(yaml_contents, "version")) return benchmark
def main(): parser = argparse.ArgumentParser( description="Converts SCAP Security Guide YAML benchmark data " "(benchmark, rules, groups) to XCCDF Shorthand Format") parser.add_argument( "--product-yaml", required=True, dest="product_yaml", help="YAML file with information about the product we are building. " "e.g.: ~/scap-security-guide/rhel7/product.yml") parser.add_argument("--bash_remediation_fns", required=True, help="XML with the XCCDF Group containing all bash " "remediation functions stored as values." "e.g.: build/bash-remediation-functions.xml") parser.add_argument("--output", required=True, help="Output XCCDF shorthand file. " "e.g.: /tmp/shorthand.xml") parser.add_argument("action", choices=["build", "list-inputs", "list-outputs"], help="Which action to perform.") args = parser.parse_args() # save a whole bunch of time if args.action == "list-outputs": print(args.output) sys.exit(0) product_yaml = ssgcommon.open_yaml(args.product_yaml) base_dir = os.path.dirname(args.product_yaml) benchmark_root = required_yaml_key(product_yaml, "benchmark_root") profiles_root = required_yaml_key(product_yaml, "profiles_root") # we have to "absolutize" the paths the right way, relative to the # product yaml path if not os.path.isabs(benchmark_root): benchmark_root = os.path.join(base_dir, benchmark_root) if not os.path.isabs(profiles_root): profiles_root = os.path.join(base_dir, profiles_root) add_from_directory(args.action, None, benchmark_root, profiles_root, args.bash_remediation_fns, args.output, product_yaml)
def from_yaml(yaml_file, product_yaml=None): yaml_contents = open_yaml(yaml_file, product_yaml) if yaml_contents is None: return None rule_id, _ = os.path.splitext(os.path.basename(yaml_file)) rule = Rule(rule_id) rule.prodtype = yaml_contents.get("prodtype", "all") rule.title = required_yaml_key(yaml_contents, "title") rule.description = required_yaml_key(yaml_contents, "description") rule.rationale = required_yaml_key(yaml_contents, "rationale") rule.severity = required_yaml_key(yaml_contents, "severity") rule.references = yaml_contents.get("references", []) rule.identifiers = yaml_contents.get("identifiers", []) rule.ocil_clause = yaml_contents.get("ocil_clause") rule.ocil = yaml_contents.get("ocil") rule.external_oval = yaml_contents.get("oval_external_content") rule.warnings = yaml_contents.get("warnings", []) for warning_list in rule.warnings: if len(warning_list) != 1: raise ValueError("Only one key/value pair should exist for each dictionary") return rule
def main(): p = argparse.ArgumentParser() p.add_argument( "--product-yaml", required=True, dest="product_yaml", help="YAML file with information about the product we are building. " "e.g.: ~/scap-security-guide/rhel7/product.yml") p.add_argument( "--remediation_type", required=True, help="language or type of the remediations we are combining." "example: ansible") p.add_argument("--build_dir", required=True, help="where is the cmake build directory. pass value of " "$CMAKE_BINARY_DIR.") p.add_argument("--output", type=argparse.FileType("wb"), required=True) p.add_argument("fixdirs", metavar="FIX_DIR", nargs="+", help="directory(ies) from which we will collect " "remediations to combine.") args, unknown = p.parse_known_args() if unknown: sys.stderr.write("Unknown positional arguments " + ",".join(unknown) + ".\n") sys.exit(1) product_yaml = open_yaml(args.product_yaml) fixcontent = ElementTree.Element( "fix-content", system="urn:xccdf:fix:script:sh", xmlns="http://checklists.nist.gov/xccdf/1.1") fixgroup = get_fixgroup_for_remediation_type(fixcontent, args.remediation_type) fixes = dict() remediation_functions = get_available_remediation_functions(args.build_dir) included_fixes_count = 0 for fixdir in args.fixdirs: try: for filename in os.listdir(fixdir): if not is_supported_filename(args.remediation_type, filename): continue # Create and populate new fix element based on shell file fixname = os.path.splitext(filename)[0] mod_file = [] config = {} fix_file_lines = process_file_with_jinja( os.path.join(fixdir, filename), product_yaml).splitlines() # Assignment automatically escapes shell characters for XML for line in fix_file_lines: line += "\n" if line.startswith('#'): try: (key, value) = line.strip('#').split('=') if key.strip() in [ 'complexity', 'disruption', 'platform', 'reboot', 'strategy' ]: config[key.strip()] = value.strip() else: if not line.startswith(FILE_GENERATED): mod_file.append(line) except ValueError: if not line.startswith(FILE_GENERATED): mod_file.append(line) else: mod_file.append(line) complexity = None disruption = None reboot = None script_platform = None strategy = None if 'complexity' in config: complexity = config['complexity'] if 'disruption' in config: disruption = config['disruption'] if 'platform' in config: script_platform = config['platform'] if 'complexity' in config: reboot = config['reboot'] if 'complexity' in config: strategy = config['strategy'] if script_platform: product_name, result = fix_is_applicable_for_product( script_platform, required_yaml_key(product_yaml, "product")) if result: if fixname in fixes: fix = fixes[fixname] for child in list(fix): fix.remove(child) else: fix = ElementTree.SubElement(fixgroup, "fix") fix.set("rule", fixname) if complexity is not None: fix.set("complexity", complexity) if disruption is not None: fix.set("disruption", disruption) if reboot is not None: fix.set("reboot", reboot) if strategy is not None: fix.set("strategy", strategy) fixes[fixname] = fix included_fixes_count += 1 fix.text = "".join(mod_file) # Expand shell variables and remediation functions # into corresponding XCCDF <sub> elements expand_xccdf_subs(fix, args.remediation_type, remediation_functions) else: sys.stderr.write("Skipping '%s' remediation script. " "The platform identifier in the " "script is missing!\n" % (filename)) except OSError as e: if e.errno != errno.ENOENT: raise else: sys.stderr.write("Not merging remediation scripts from the " "'%s' directory as the directory does not " "exist.\n" % (fixdir)) sys.stderr.write("Merged %d %s remediations.\n" % (included_fixes_count, args.remediation_type)) tree = ElementTree.ElementTree(fixcontent) tree.write(args.output) sys.exit(0)
def main(): p = argparse.ArgumentParser() p.add_argument("--ssg_version", default="unknown", help="SSG version for reporting purposes. example: 0.1.34") p.add_argument( "--product-yaml", required=True, dest="product_yaml", help="YAML file with information about the product we are building. " "e.g.: ~/scap-security-guide/rhel7/product.yml" ) p.add_argument("--oval_config", required=True, help="Location of the oval.config file.") p.add_argument("--oval_version", required=True, help="OVAL version to use. Example: 5.11, 5.10, ...") p.add_argument("--output", type=argparse.FileType("wb"), required=True) p.add_argument("ovaldirs", metavar="OVAL_DIR", nargs="+", help="Directory(ies) from which we will collect " "OVAL definitions to combine. Order matters, latter " "directories override former.") args, unknown = p.parse_known_args() if unknown: sys.stderr.write( "Unknown positional arguments " + ",".join(unknown) + ".\n" ) sys.exit(1) product_yaml = ssgcommon.open_yaml(args.product_yaml) if os.path.isfile(args.oval_config): multi_platform = parse_conf_file( args.oval_config, ssgcommon.required_yaml_key(product_yaml, "product") ) header = ssgcommon.oval_generated_header("combine-ovals.py", args.oval_version, args.ssg_version) else: sys.stderr.write("The directory specified does not contain the %s " "file!\n" % (args.oval_config)) sys.exit(1) body = checks(product_yaml, args.oval_version, args.ovaldirs) # parse new file(string) as an ElementTree, so we can reorder elements # appropriately corrected_tree = ElementTree.fromstring( ("%s%s%s" % (header, body, footer)).encode("utf-8")) tree = add_platforms(corrected_tree, multi_platform) definitions = ElementTree.Element("{%s}definitions" % oval_ns) tests = ElementTree.Element("{%s}tests" % oval_ns) objects = ElementTree.Element("{%s}objects" % oval_ns) states = ElementTree.Element("{%s}states" % oval_ns) variables = ElementTree.Element("{%s}variables" % oval_ns) for childnode in tree.findall("./{%s}def-group/*" % oval_ns): if childnode.tag is ElementTree.Comment: continue elif childnode.tag.endswith("definition"): append(definitions, childnode) elif childnode.tag.endswith("_test"): append(tests, childnode) elif childnode.tag.endswith("_object"): append(objects, childnode) elif childnode.tag.endswith("_state"): append(states, childnode) elif childnode.tag.endswith("_variable"): append(variables, childnode) else: sys.stderr.write("Warning: Unknown element '%s'\n" % (childnode.tag)) root = ElementTree.fromstring(("%s%s" % (header, footer)).encode("utf-8")) root.append(definitions) root.append(tests) root.append(objects) root.append(states) if list(variables): root.append(variables) ElementTree.ElementTree(root).write(args.output) sys.exit(0)