def find_rules(directory, func): # Iterates over passed directory to correctly parse rules (which are # YAML files with internal macros). The most recently seen product.yml # takes precedence over previous product.yml, e.g.: # # a/product.yml # a/b/product.yml -- will be selected for the following rule: # a/b/c/something.rule # # The corresponding rule and contents of the product.yml are then passed # into func(/path/to/rule, product_yaml_contents); if the result evaluates # to true, the tuple (/path/to/rule, /path/to/product.yml) is saved as a # result. # # This process mimics the build system and allows us to find rule files # which satisfy the constraints of the passed func. results = [] product_yamls = {} product_yaml_paths = {} product_yaml = None product_yaml_path = None for root, dirs, files in os.walk(directory): dirs.sort() files.sort() if "product.yml" in files: product_yaml_path = os.path.join(root, "product.yml") product_yaml = yaml.open_raw(product_yaml_path) product_yamls[root] = product_yaml product_yaml_paths[root] = product_yaml_path # for d in dirs: # product_yamls[os.path.join(root, d)] = product_yaml # product_yaml_paths[os.path.join(root, d)] = product_yaml_path elif root in product_yamls: product_yaml = product_yamls[root] product_yaml_path = product_yaml_paths[root] # for d in dirs: # product_yamls[os.path.join(root, d)] = product_yaml # product_yaml_paths[os.path.join(root, d)] = product_yaml_path else: pass for filename in files: path = os.path.join(root, filename) rule_filename_id = 'rule.yml' rule_filename_id_len = len(rule_filename_id) if len(path) < rule_filename_id_len \ or path[-(rule_filename_id_len):] != rule_filename_id \ or "tests/" in path: continue try: if func(path, product_yaml): results.append((path, product_yaml_path)) except jinja2.exceptions.UndefinedError: print( "Failed to parse file %s (with product.yml: %s). Skipping" % (path, product_yaml_path)) pass return results
def find_rules(directory, func): # Iterates over passed directory to correctly parse rules (which are # YAML files with internal macros). The most recently seen product.yml # takes precedence over previous product.yml, e.g.: # # a/product.yml # a/b/product.yml -- will be selected for the following rule: # a/b/c/something.rule # # The corresponding rule and contents of the product.yml are then passed # into func(/path/to/rule, product_yaml_contents); if the result evaluates # to true, the tuple (/path/to/rule, /path/to/product.yml) is saved as a # result. # # This process mimics the build system and allows us to find rule files # which satisfy the constraints of the passed func. results = [] product_yamls = {} product_yaml_paths = {} product_yaml = None product_yaml_path = None for root, dirs, files in os.walk(directory): if "product.yml" in files: product_yaml_path = os.path.join(root, "product.yml") product_yaml = yaml.open_raw(product_yaml_path) product_yamls[root] = product_yaml product_yaml_paths[root] = product_yaml_path # for d in dirs: # product_yamls[os.path.join(root, d)] = product_yaml # product_yaml_paths[os.path.join(root, d)] = product_yaml_path elif root in product_yamls: product_yaml = product_yamls[root] product_yaml_path = product_yaml_paths[root] # for d in dirs: # product_yamls[os.path.join(root, d)] = product_yaml # product_yaml_paths[os.path.join(root, d)] = product_yaml_path else: pass for filename in files: path = os.path.join(root, filename) rule_filename_id = 'rule.yml' rule_filename_id_len = len(rule_filename_id) if len(path) < rule_filename_id_len \ or path[-(rule_filename_id_len):] != rule_filename_id \ or "tests/" in path: continue try: if func(path, product_yaml): results.append((path, product_yaml_path)) except jinja2.exceptions.UndefinedError: print("Failed to parse file %s (with product.yaml: %s). Skipping" % (path, product_yaml_path)) pass return results
def find_int_references(directory): results = find_rules(directory, has_int_reference) print("Number of rules with integer references: %d" % len(results)) for result in results: rule_path = result[0] product_yaml_path = result[1] product_yaml = None if product_yaml_path is not None: product_yaml = yaml.open_raw(product_yaml_path) fix_file(rule_path, product_yaml, fix_int_reference)
def fix_empty_identifiers(directory): results = find_rules(directory, has_empty_identifier) print("Number of rules with empty identifiers: %d" % len(results)) for result in results: rule_path = result[0] product_yaml_path = result[1] product_yaml = None if product_yaml_path is not None: product_yaml = yaml.open_raw(product_yaml_path) fix_file(rule_path, product_yaml, fix_empty_identifier)
def find_int_references(directory): results = find_rules(directory, has_int_reference) print("Number of rules with integer references: %d" % len(results)) for result in results: rule_path = result[0] product_yaml_path = result[1] product_yaml = None if product_yaml_path is not None: product_yaml = yaml.open_raw(product_yaml_path) fix_file(rule_path, product_yaml, fix_int_reference)
def fix_empty_identifiers(directory): results = find_rules(directory, has_empty_identifier) print("Number of rules with empty identifiers: %d" % len(results)) for result in results: rule_path = result[0] product_yaml_path = result[1] product_yaml = None if product_yaml_path is not None: product_yaml = yaml.open_raw(product_yaml_path) fix_file(rule_path, product_yaml, fix_empty_identifier)
def test_product_cpes(): # CPEs are loaded from `DATADIR/product.yml` but also from # `DATADIR/applicability` because `DATADIR/product.yml` references the # `DATADIR/applicability` directory in the `cpes_root` key product_yaml_path = os.path.join(DATADIR, "product.yml") product_yaml = open_raw(product_yaml_path) product_yaml["product_dir"] = os.path.dirname(product_yaml_path) product_cpes = ssg.build_cpe.ProductCPEs(product_yaml) # get a product CPE by name and verify it's loaded # this CPE is defined in `DATADIR/product.yml` rhel7_cpe = product_cpes.get_cpe("rhel7") assert (rhel7_cpe.name == "cpe:/o:redhat:enterprise_linux:7") assert (rhel7_cpe.title == "Red Hat Enterprise Linux 7") assert (rhel7_cpe.check_id == "installed_OS_is_rhel7") assert (rhel7_cpe.bash_conditional == "") assert (rhel7_cpe.ansible_conditional == "") # get CPE by ID and verify it's loaded, the get_cpe method should return # the same object as when CPE name was used above rhel7_cpe_2 = product_cpes.get_cpe("cpe:/o:redhat:enterprise_linux:7") assert (rhel7_cpe_2.name == rhel7_cpe.name) assert (rhel7_cpe_2.title == rhel7_cpe_2.title) assert (rhel7_cpe_2.check_id == rhel7_cpe.check_id) assert (rhel7_cpe_2.bash_conditional == rhel7_cpe.bash_conditional) assert (rhel7_cpe_2.ansible_conditional == rhel7_cpe.ansible_conditional) # get a content CPE by name and verify it's loaded # this CPE is defined in `DATADIR/applicability/virtualization.yml` cpe1 = product_cpes.get_cpe("machine") assert (cpe1.name == "cpe:/a:machine") assert (cpe1.title == "Bare-metal or Virtual Machine") assert (cpe1.check_id == "installed_env_is_a_machine") assert ( cpe1.ansible_conditional == "ansible_virtualization_type not in [\"docker\", \"lxc\", \"openvz\", \"podman\", \"container\"]" ) assert (cpe1.bash_conditional == "[ ! -f /.dockerenv ] && [ ! -f /run/.containerenv ]") # get CPE by ID and verify it's loaded, the get_cpe method should return # the same object as when CPE name was used above cpe2 = product_cpes.get_cpe("cpe:/a:machine") assert (cpe2.name == cpe1.name) assert (cpe2.title == cpe1.title) assert (cpe2.check_id == cpe1.check_id) assert (cpe2.ansible_conditional == cpe1.ansible_conditional) assert (cpe2.bash_conditional == cpe1.bash_conditional)
def fix_empty_identifiers(args, product_yaml): results = find_rules(args, has_empty_identifier) print("Number of rules with empty identifiers: %d" % len(results)) for result in results: rule_path = result[0] product_yaml_path = result[2] if product_yaml_path is not None: product_yaml = yaml.open_raw(product_yaml_path) if args.dry_run: print(rule_path + " has one or more empty identifiers") continue fix_file_prompt(rule_path, product_yaml, fix_empty_identifier, args) exit(int(len(results) > 0))
def product_cpes(): product_yaml_path = os.path.join(DATADIR, "product.yml") product_yaml = open_raw(product_yaml_path) product_yaml["product_dir"] = os.path.dirname(product_yaml_path) return ProductCPEs(product_yaml)