def template_tests(product=None):
"""
Create a temporary directory with test cases parsed via jinja using
product-specific context.
"""
# Set up an empty temp directory
tmpdir = tempfile.mkdtemp()
# We want to remove the temporary directory on failure, but preserve
# it on success. Wrap in a try/except block and reraise the original
# exception after removing the temporary directory.
try:
# Load the product context we're executing under, if any.
product_yaml = get_product_context(product)
# Initialize a mock template_builder.
empty = "/ssgts/empty/placeholder"
template_builder = ssg.templates.Builder(product_yaml, empty,
_SHARED_TEMPLATES, empty,
empty)
# Note that we're not exactly copying 1-for-1 the contents of the
# directory structure into the temporary one. Instead we want a
# flattened mapping with all rules in a single top-level directory
# and all tests immediately contained within it. That is:
#
# /group_a/rule_a/tests/something.pass.sh -> /rule_a/something.pass.sh
for dirpath, dirnames, _ in walk_through_benchmark_dirs(product):
# Skip anything that isn't obviously a rule.
if not is_rule_dir(dirpath):
continue
template_rule_tests(product, product_yaml, template_builder,
tmpdir, dirpath)
except Exception as exp:
shutil.rmtree(tmpdir, ignore_errors=True)
raise exp
return tmpdir
def iterate_over_rules(product=None):
"""Iterate over rule directories which have test scenarios".
Returns:
Named tuple Rule having these fields:
directory -- absolute path to the rule "tests" subdirectory
containing the test scenarios in Bash
id -- full rule id as it is present in datastream
short_id -- short rule ID, the same as basename of the directory
containing the test scenarios in Bash
files -- list of executable .sh files in the uploaded tarball
"""
# Here we need to perform some magic to handle parsing the rule (from a
# product perspective) and loading any templated tests. In particular,
# identifying which tests to potentially run involves invoking the
# templating engine.
#
# Begin by loading context about our execution environment, if any.
product_yaml = get_product_context(product)
# Initialize a mock template_builder.
empty = "/ssgts/empty/placeholder"
template_builder = ssg.templates.Builder(product_yaml, empty,
_SHARED_TEMPLATES, empty, empty)
for dirpath, dirnames, filenames in walk_through_benchmark_dirs(product):
if is_rule_dir(dirpath):
short_rule_id = os.path.basename(dirpath)
# Load the rule itself to check for a template.
rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml,
product)
template_name = None
# Before we get too far, we wish to search the rule YAML to see if
# it is applicable to the current product. If we have a product
# and the rule isn't applicable for the product, there's no point
# in continuing with the rest of the loading. This should speed up
# the loading of the templated tests. Note that we've already
# parsed the prodtype into local_env_yaml
if product and local_env_yaml['products']:
prodtypes = local_env_yaml['products']
if "all" not in prodtypes and product not in prodtypes:
continue
# All tests is a mapping from path (in the tarball) to contents
# of the test case. This is necessary because later code (which
# attempts to parse headers from the test case) don't have easy
# access to templated content. By reading it and returning it
# here, we can save later code from having to understand the
# templating system.
all_tests = dict()
# Start by checking for templating tests and provision them if
# present.
if rule.template and rule.template['vars']:
templated_tests = template_builder.get_all_tests(
rule.id_, rule.template, local_env_yaml)
all_tests.update(templated_tests)
template_name = rule.template['name']
# Add additional tests from the local rule directory. Note that,
# like the behavior in template_tests, this will overwrite any
# templated tests with the same file name.
tests_dir = os.path.join(dirpath, "tests")
if os.path.exists(tests_dir):
tests_dir_files = os.listdir(tests_dir)
for test_case in tests_dir_files:
test_path = os.path.join(tests_dir, test_case)
if os.path.isdir(test_path):
continue
all_tests[test_case] = process_file_with_macros(
test_path, local_env_yaml)
# Filter out everything except the shell test scenarios.
# Other files in rule directories are editor swap files
# or other content than a test case.
allowed_scripts = filter(lambda x: x.endswith(".sh"), all_tests)
content_mapping = {x: all_tests[x] for x in allowed_scripts}
# Skip any rules that lack any content. This ensures that if we
# end up with rules with a template lacking tests and without any
# rule directory tests, we don't include the empty rule here.
if not content_mapping:
continue
full_rule_id = OSCAP_RULE + short_rule_id
result = Rule(directory=tests_dir,
id=full_rule_id,
short_id=short_rule_id,
files=content_mapping,
template=template_name)
yield result
def template_tests(product=None):
"""
Create a temporary directory with test cases parsed via jinja using
product-specific context.
"""
# Set up an empty temp directory
tmpdir = tempfile.mkdtemp()
# We want to remove the temporary directory on failure, but preserve
# it on success. Wrap in a try/except block and reraise the original
# exception after removing the temporary directory.
try:
# Load product's YAML file if present. This will allow us to parse
# tests in the context of the product we're executing under.
product_yaml = dict()
if product:
yaml_path = product_yaml_path(SSG_ROOT, product)
product_yaml = load_product_yaml(yaml_path)
# Below we could run into a DocumentationNotComplete error. However,
# because the test suite isn't executed in the context of a particular
# build (though, ideally it would be linked), we may not know exactly
# whether the top-level rule/profile we're testing is actually
# completed. Thus, forcibly set the required property to bypass this
# error.
product_yaml['cmake_build_type'] = 'Debug'
# Note that we're not exactly copying 1-for-1 the contents of the
# directory structure into the temporary one. Instead we want a
# flattened mapping with all rules in a single top-level directory
# and all tests immediately contained within it. That is:
#
# /group_a/rule_a/tests/something.pass.sh -> /rule_a/something.pass.sh
for dirpath, dirnames, _ in walk_through_benchmark_dirs(product):
# Skip anything that isn't obviously a rule.
if "tests" not in dirnames or not is_rule_dir(dirpath):
continue
# Load rule content in our environment. We use this to satisfy
# some implied properties that might be used in the test suite.
rule_path = get_rule_dir_yaml(dirpath)
rule = RuleYAML.from_yaml(rule_path, product_yaml)
# Note that most places would check prodtype, but we don't care
# about that here: if the rule is available to the product, we
# load and parse it anyways as we have no knowledge of the
# top-level profile or rule passed into the test suite.
prodtypes = parse_prodtype(rule.prodtype)
# Our local copy of env_yaml needs some properties from rule.yml
# for completeness.
local_env_yaml = dict()
local_env_yaml.update(product_yaml)
local_env_yaml['rule_id'] = rule.id_
local_env_yaml['rule_title'] = rule.title
local_env_yaml['products'] = prodtypes
# Create the destination directory.
dest_path = os.path.join(tmpdir, rule.id_)
os.mkdir(dest_path)
# Walk the test directory, writing all tests into the output
# directory, recursively.
tests_dir_path = os.path.join(dirpath, "tests")
tests_dir_path = os.path.abspath(tests_dir_path)
for dirpath, dirnames, filenames in os.walk(tests_dir_path):
for dirname in dirnames:
# We want to recreate the correct path under the temporary
# directory. Resolve it to a relative path from the tests/
# directory.
dir_path = _rel_abs_path(os.path.join(dirpath, dirname),
tests_dir_path)
assert '../' not in dir_path
tmp_dir_path = os.path.join(dest_path, dir_path)
os.mkdir(tmp_dir_path)
for filename in filenames:
# We want to recreate the correct path under the temporary
# directory. Resolve it to a relative path from the tests/
# directory. Assumption: directories should be created
# prior to recursing into them, so we don't need to handle
# if a file's parent directory doesn't yet exist under the
# destination.
src_test_path = os.path.join(dirpath, filename)
rel_test_path = _rel_abs_path(src_test_path,
tests_dir_path)
dest_test_path = os.path.join(dest_path, rel_test_path)
# Rather than performing an OS-level copy, we need to
# first parse the test with jinja and then write it back
# out to the destination.
parsed_test = process_file(src_test_path, local_env_yaml)
with open(dest_test_path, 'w') as output_fp:
print(parsed_test, file=output_fp)
except Exception as exp:
shutil.rmtree(tmpdir, ignore_errors=True)
raise exp
return tmpdir
