def find_rules(directory, func):
results = []
product_yamls = {}
product_yaml_paths = {}
for root, dirs, files in os.walk(directory):
product_yaml = None
product_yaml_path = None
if "product.yml" in files:
product_yaml_path = os.path.join(root, "product.yml")
product_yaml = ssgcommon.open_yaml(product_yaml_path)
product_yamls[root] = product_yaml
product_yaml_paths[root] = product_yaml_path
for d in dirs:
product_yamls[os.path.join(root, d)] = product_yaml
product_yaml_paths[os.path.join(root, d)] = product_yaml_path
elif root in product_yamls:
product_yaml = product_yamls[root]
product_yaml_path = product_yaml_paths[root]
for d in dirs:
product_yamls[os.path.join(root, d)] = product_yaml
product_yaml_paths[os.path.join(root, d)] = product_yaml_path
else:
pass
# print("No product yaml for file: %s" % root)
for filename in files:
path = os.path.join(root, filename)
if len(path) < 5 or path[-5:] != '.rule':
continue
if func(path, product_yaml):
results.append((path, product_yaml_path))
return results
def find_rules(directory, func):
# Iterates over passed directory to correctly parse rules (which are
# YAML files with internal macros). The most recently seen product.yml
# takes precedence over previous product.yml, e.g.:
#
# a/product.yml
# a/b/product.yml -- will be selected for the following rule:
# a/b/c/something.rule
#
# The corresponding rule and contents of the product.yml are then passed
# into func(/path/to/rule, product_yaml_contents); if the result evalutes
# to true, the tuple (/path/to/rule, /path/to/product.yml) is saved as a
# result.
#
# This process mimics the build system and allows us to find rule files
# which satisfy the constraints of the passed func.
results = []
product_yamls = {}
product_yaml_paths = {}
for root, dirs, files in os.walk(directory):
product_yaml = None
product_yaml_path = None
if "product.yml" in files:
product_yaml_path = os.path.join(root, "product.yml")
product_yaml = ssgcommon.open_yaml(product_yaml_path)
product_yamls[root] = product_yaml
product_yaml_paths[root] = product_yaml_path
for d in dirs:
product_yamls[os.path.join(root, d)] = product_yaml
product_yaml_paths[os.path.join(root, d)] = product_yaml_path
elif root in product_yamls:
product_yaml = product_yamls[root]
product_yaml_path = product_yaml_paths[root]
for d in dirs:
product_yamls[os.path.join(root, d)] = product_yaml
product_yaml_paths[os.path.join(root, d)] = product_yaml_path
else:
pass
for filename in files:
path = os.path.join(root, filename)
if len(path) < 5 or path[-5:] != '.rule':
continue
try:
if func(path, product_yaml):
results.append((path, product_yaml_path))
except jinja2.exceptions.UndefinedError:
print(
"Failed to parse file %s (with product.yaml: %s). Skipping"
% (path, product_yaml_path))
pass
return results
def from_yaml(yaml_file, product_yaml=None):
yaml_contents = open_yaml(yaml_file, product_yaml)
if yaml_contents is None:
return None
group_id, _ = os.path.splitext(os.path.basename(yaml_file))
group = Group(group_id)
group.prodtype = yaml_contents.get("prodtype", "all")
group.title = required_yaml_key(yaml_contents, "title")
group.description = required_yaml_key(yaml_contents, "description")
return group
def from_yaml(yaml_file, product_yaml=None):
yaml_contents = open_yaml(yaml_file, product_yaml)
if yaml_contents is None:
return None
value_id, _ = os.path.splitext(os.path.basename(yaml_file))
value = Value(value_id)
value.title = required_yaml_key(yaml_contents, "title")
value.description = required_yaml_key(yaml_contents, "description")
value.type = required_yaml_key(yaml_contents, "type")
value.options = required_yaml_key(yaml_contents, "options")
return value
def fix_empty_identifiers(directory):
results = find_rules(directory, has_empty_identifier)
print("Number of rules with empty identifiers: %d" % len(results))
for result in results:
rule_path = result[0]
product_yaml_path = result[1]
product_yaml = None
if product_yaml_path is not None:
product_yaml = ssgcommon.open_yaml(product_yaml_path)
fix_empty_identifier(rule_path, product_yaml)
def find_int_references(directory):
results = find_rules(directory, has_int_reference)
print("Number of rules with integer references: %d" % len(results))
for result in results:
rule_path = result[0]
product_yaml_path = result[1]
product_yaml = None
if product_yaml_path is not None:
product_yaml = ssgcommon.open_yaml(product_yaml_path)
fix_file(rule_path, product_yaml, fix_int_reference)
def from_yaml(yaml_file, product_yaml=None):
yaml_contents = open_yaml(yaml_file, product_yaml)
if yaml_contents is None:
return None
basename, _ = os.path.splitext(os.path.basename(yaml_file))
profile = Profile(basename)
profile.title = required_yaml_key(yaml_contents, "title")
profile.description = required_yaml_key(yaml_contents, "description")
profile.extends = yaml_contents.get("extends", None)
profile.selections = required_yaml_key(yaml_contents, "selections")
return profile
def find_prefix_cce(directory):
results = find_rules(directory, has_prefix_cce)
print("Number of rules with prefixed CCEs: %d" % len(results))
for result in results:
rule_path = result[0]
rule_path = result[0]
product_yaml_path = result[1]
product_yaml = None
if product_yaml_path is not None:
product_yaml = ssgcommon.open_yaml(product_yaml_path)
fix_file(rule_path, product_yaml, fix_prefix_cce)
def from_yaml(yaml_file, product_yaml=None):
yaml_contents = open_yaml(yaml_file, product_yaml)
if yaml_contents is None:
return None
group_id, _ = os.path.splitext(os.path.basename(yaml_file))
group = Group(group_id)
group.prodtype = yaml_contents.get("prodtype", "all")
group.title = required_yaml_key(yaml_contents, "title")
group.description = required_yaml_key(yaml_contents, "description")
group.warnings = yaml_contents.get("warnings", [])
for warning_list in group.warnings:
if len(warning_list) != 1:
raise ValueError("Only one key/value pair should exist for each dictionary")
return group
def from_yaml(yaml_file, product_yaml=None):
yaml_contents = open_yaml(yaml_file, product_yaml)
if yaml_contents is None:
return None
value_id, _ = os.path.splitext(os.path.basename(yaml_file))
value = Value(value_id)
value.title = required_yaml_key(yaml_contents, "title")
value.description = required_yaml_key(yaml_contents, "description")
value.type = required_yaml_key(yaml_contents, "type")
value.options = required_yaml_key(yaml_contents, "options")
value.warnings = yaml_contents.get("warnings", [])
for warning_list in value.warnings:
if len(warning_list) != 1:
raise ValueError("Only one key/value pair should exist for each dictionary")
return value
def from_yaml(yaml_file, product_yaml=None):
yaml_contents = open_yaml(yaml_file, product_yaml)
if yaml_contents is None:
return None
rule_id, _ = os.path.splitext(os.path.basename(yaml_file))
rule = Rule(rule_id)
rule.prodtype = yaml_contents.get("prodtype", "all")
rule.title = required_yaml_key(yaml_contents, "title")
rule.description = required_yaml_key(yaml_contents, "description")
rule.rationale = required_yaml_key(yaml_contents, "rationale")
rule.severity = required_yaml_key(yaml_contents, "severity")
rule.references = yaml_contents.get("references", [])
rule.identifiers = yaml_contents.get("identifiers", [])
rule.ocil_clause = yaml_contents.get("ocil_clause")
rule.ocil = yaml_contents.get("ocil")
rule.external_oval = yaml_contents.get("oval_external_content")
return rule
def from_yaml(yaml_file, id_, product_yaml=None):
yaml_contents = open_yaml(yaml_file, product_yaml)
if yaml_contents is None:
return None
benchmark = Benchmark(id_)
benchmark.title = required_yaml_key(yaml_contents, "title")
benchmark.status = required_yaml_key(yaml_contents, "status")
benchmark.description = required_yaml_key(yaml_contents, "description")
notice_contents = required_yaml_key(yaml_contents, "notice")
benchmark.notice_id = required_yaml_key(notice_contents, "id")
benchmark.notice_description = required_yaml_key(
notice_contents, "description")
benchmark.front_matter = required_yaml_key(yaml_contents,
"front-matter")
benchmark.rear_matter = required_yaml_key(yaml_contents, "rear-matter")
benchmark.cpes = yaml_contents.get("cpes", [])
benchmark.version = str(required_yaml_key(yaml_contents, "version"))
return benchmark
def main():
parser = argparse.ArgumentParser(
description="Converts SCAP Security Guide YAML benchmark data "
"(benchmark, rules, groups) to XCCDF Shorthand Format")
parser.add_argument(
"--product-yaml",
required=True,
dest="product_yaml",
help="YAML file with information about the product we are building. "
"e.g.: ~/scap-security-guide/rhel7/product.yml")
parser.add_argument("--bash_remediation_fns",
required=True,
help="XML with the XCCDF Group containing all bash "
"remediation functions stored as values."
"e.g.: build/bash-remediation-functions.xml")
parser.add_argument("--output",
required=True,
help="Output XCCDF shorthand file. "
"e.g.: /tmp/shorthand.xml")
parser.add_argument("action",
choices=["build", "list-inputs", "list-outputs"],
help="Which action to perform.")
args = parser.parse_args()
# save a whole bunch of time
if args.action == "list-outputs":
print(args.output)
sys.exit(0)
product_yaml = ssgcommon.open_yaml(args.product_yaml)
base_dir = os.path.dirname(args.product_yaml)
benchmark_root = required_yaml_key(product_yaml, "benchmark_root")
profiles_root = required_yaml_key(product_yaml, "profiles_root")
# we have to "absolutize" the paths the right way, relative to the
# product yaml path
if not os.path.isabs(benchmark_root):
benchmark_root = os.path.join(base_dir, benchmark_root)
if not os.path.isabs(profiles_root):
profiles_root = os.path.join(base_dir, profiles_root)
add_from_directory(args.action, None, benchmark_root, profiles_root,
args.bash_remediation_fns, args.output, product_yaml)
def from_yaml(yaml_file, product_yaml=None):
yaml_contents = open_yaml(yaml_file, product_yaml)
if yaml_contents is None:
return None
rule_id, _ = os.path.splitext(os.path.basename(yaml_file))
rule = Rule(rule_id)
rule.prodtype = yaml_contents.get("prodtype", "all")
rule.title = required_yaml_key(yaml_contents, "title")
rule.description = required_yaml_key(yaml_contents, "description")
rule.rationale = required_yaml_key(yaml_contents, "rationale")
rule.severity = required_yaml_key(yaml_contents, "severity")
rule.references = yaml_contents.get("references", [])
rule.identifiers = yaml_contents.get("identifiers", [])
rule.ocil_clause = yaml_contents.get("ocil_clause")
rule.ocil = yaml_contents.get("ocil")
rule.external_oval = yaml_contents.get("oval_external_content")
rule.warnings = yaml_contents.get("warnings", [])
for warning_list in rule.warnings:
if len(warning_list) != 1:
raise ValueError("Only one key/value pair should exist for each dictionary")
return rule
def main():
p = argparse.ArgumentParser()
p.add_argument(
"--product-yaml",
required=True,
dest="product_yaml",
help="YAML file with information about the product we are building. "
"e.g.: ~/scap-security-guide/rhel7/product.yml")
p.add_argument(
"--remediation_type",
required=True,
help="language or type of the remediations we are combining."
"example: ansible")
p.add_argument("--build_dir",
required=True,
help="where is the cmake build directory. pass value of "
"$CMAKE_BINARY_DIR.")
p.add_argument("--output", type=argparse.FileType("wb"), required=True)
p.add_argument("fixdirs",
metavar="FIX_DIR",
nargs="+",
help="directory(ies) from which we will collect "
"remediations to combine.")
args, unknown = p.parse_known_args()
if unknown:
sys.stderr.write("Unknown positional arguments " + ",".join(unknown) +
".\n")
sys.exit(1)
product_yaml = open_yaml(args.product_yaml)
fixcontent = ElementTree.Element(
"fix-content",
system="urn:xccdf:fix:script:sh",
xmlns="http://checklists.nist.gov/xccdf/1.1")
fixgroup = get_fixgroup_for_remediation_type(fixcontent,
args.remediation_type)
fixes = dict()
remediation_functions = get_available_remediation_functions(args.build_dir)
included_fixes_count = 0
for fixdir in args.fixdirs:
try:
for filename in os.listdir(fixdir):
if not is_supported_filename(args.remediation_type, filename):
continue
# Create and populate new fix element based on shell file
fixname = os.path.splitext(filename)[0]
mod_file = []
config = {}
fix_file_lines = process_file_with_jinja(
os.path.join(fixdir, filename), product_yaml).splitlines()
# Assignment automatically escapes shell characters for XML
for line in fix_file_lines:
line += "\n"
if line.startswith('#'):
try:
(key, value) = line.strip('#').split('=')
if key.strip() in [
'complexity', 'disruption', 'platform',
'reboot', 'strategy'
]:
config[key.strip()] = value.strip()
else:
if not line.startswith(FILE_GENERATED):
mod_file.append(line)
except ValueError:
if not line.startswith(FILE_GENERATED):
mod_file.append(line)
else:
mod_file.append(line)
complexity = None
disruption = None
reboot = None
script_platform = None
strategy = None
if 'complexity' in config:
complexity = config['complexity']
if 'disruption' in config:
disruption = config['disruption']
if 'platform' in config:
script_platform = config['platform']
if 'complexity' in config:
reboot = config['reboot']
if 'complexity' in config:
strategy = config['strategy']
if script_platform:
product_name, result = fix_is_applicable_for_product(
script_platform,
required_yaml_key(product_yaml, "product"))
if result:
if fixname in fixes:
fix = fixes[fixname]
for child in list(fix):
fix.remove(child)
else:
fix = ElementTree.SubElement(fixgroup, "fix")
fix.set("rule", fixname)
if complexity is not None:
fix.set("complexity", complexity)
if disruption is not None:
fix.set("disruption", disruption)
if reboot is not None:
fix.set("reboot", reboot)
if strategy is not None:
fix.set("strategy", strategy)
fixes[fixname] = fix
included_fixes_count += 1
fix.text = "".join(mod_file)
# Expand shell variables and remediation functions
# into corresponding XCCDF <sub> elements
expand_xccdf_subs(fix, args.remediation_type,
remediation_functions)
else:
sys.stderr.write("Skipping '%s' remediation script. "
"The platform identifier in the "
"script is missing!\n" % (filename))
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
sys.stderr.write("Not merging remediation scripts from the "
"'%s' directory as the directory does not "
"exist.\n" % (fixdir))
sys.stderr.write("Merged %d %s remediations.\n" %
(included_fixes_count, args.remediation_type))
tree = ElementTree.ElementTree(fixcontent)
tree.write(args.output)
sys.exit(0)
def main():
p = argparse.ArgumentParser()
p.add_argument("--ssg_version", default="unknown",
help="SSG version for reporting purposes. example: 0.1.34")
p.add_argument(
"--product-yaml", required=True, dest="product_yaml",
help="YAML file with information about the product we are building. "
"e.g.: ~/scap-security-guide/rhel7/product.yml"
)
p.add_argument("--oval_config", required=True,
help="Location of the oval.config file.")
p.add_argument("--oval_version", required=True,
help="OVAL version to use. Example: 5.11, 5.10, ...")
p.add_argument("--output", type=argparse.FileType("wb"), required=True)
p.add_argument("ovaldirs", metavar="OVAL_DIR", nargs="+",
help="Directory(ies) from which we will collect "
"OVAL definitions to combine. Order matters, latter "
"directories override former.")
args, unknown = p.parse_known_args()
if unknown:
sys.stderr.write(
"Unknown positional arguments " + ",".join(unknown) + ".\n"
)
sys.exit(1)
product_yaml = ssgcommon.open_yaml(args.product_yaml)
if os.path.isfile(args.oval_config):
multi_platform = parse_conf_file(
args.oval_config,
ssgcommon.required_yaml_key(product_yaml, "product")
)
header = ssgcommon.oval_generated_header("combine-ovals.py", args.oval_version, args.ssg_version)
else:
sys.stderr.write("The directory specified does not contain the %s "
"file!\n" % (args.oval_config))
sys.exit(1)
body = checks(product_yaml, args.oval_version, args.ovaldirs)
# parse new file(string) as an ElementTree, so we can reorder elements
# appropriately
corrected_tree = ElementTree.fromstring(
("%s%s%s" % (header, body, footer)).encode("utf-8"))
tree = add_platforms(corrected_tree, multi_platform)
definitions = ElementTree.Element("{%s}definitions" % oval_ns)
tests = ElementTree.Element("{%s}tests" % oval_ns)
objects = ElementTree.Element("{%s}objects" % oval_ns)
states = ElementTree.Element("{%s}states" % oval_ns)
variables = ElementTree.Element("{%s}variables" % oval_ns)
for childnode in tree.findall("./{%s}def-group/*" % oval_ns):
if childnode.tag is ElementTree.Comment:
continue
elif childnode.tag.endswith("definition"):
append(definitions, childnode)
elif childnode.tag.endswith("_test"):
append(tests, childnode)
elif childnode.tag.endswith("_object"):
append(objects, childnode)
elif childnode.tag.endswith("_state"):
append(states, childnode)
elif childnode.tag.endswith("_variable"):
append(variables, childnode)
else:
sys.stderr.write("Warning: Unknown element '%s'\n"
% (childnode.tag))
root = ElementTree.fromstring(("%s%s" % (header, footer)).encode("utf-8"))
root.append(definitions)
root.append(tests)
root.append(objects)
root.append(states)
if list(variables):
root.append(variables)
ElementTree.ElementTree(root).write(args.output)
sys.exit(0)
