def test_get_results_pass(conformity_report):
"""
GIVEN `get_results` is called
WHEN no offending entries are found
THEN return empty list (no security issues found)
"""
c = CcValidator()
report = c.get_results(conformity_report)
assert not report
def test_read_template_file_valid_file():
"""
GIVEN `read_template_file` is called
WHEN a file is provided
THEN return the file contents as a string
"""
c = CcValidator()
template_str = c.read_template_file()
assert "Resources" in template_str
def test_get_results_fail(monkeypatch, conformity_report):
"""
GIVEN `get_results` is called
WHEN offending entries are found
THEN return a `list` of failed entries
"""
monkeypatch.setenv("CC_RISK_LEVEL", "LOW")
c = CcValidator()
report = c.get_results(conformity_report)
assert report
def test_generate_payload():
"""
GIVEN `generate_payload` is called
WHEN a valid template is provided
THEN return the payload which will be sent to Conformity
"""
c = CcValidator()
template_str = c.read_template_file()
payload = c.generate_payload(template_str)
assert "data" in payload
def test_run_validation_no_profile_id():
"""
GIVEN `run_validation` is called
WHEN no profile ID is provided
THEN return the payload provided by Conformity
"""
c = CcValidator()
template_str = c.read_template_file()
payload = c.generate_payload(template_str)
validation = c.run_validation(payload)
assert "data" in validation
def test_fail_pipeline_enabled():
"""
GIVEN `_fail_pipeline` is called
WHEN `FAIL_PIPELINE` is anything other than `disabled`
THEN return `False` (pipeline will fail when issues are found)
"""
c = CcValidator()
# real template not required for testing
fail_pipeline = c._fail_pipeline("")
assert fail_pipeline is True
def test_run_secure_template(caplog):
"""
GIVEN a valid template is passed in
WHEN the template has no security issues
THEN exit with a code of 0
"""
caplog.set_level(logging.DEBUG)
c = CcValidator()
with pytest.raises(SystemExit):
c.run()
assert "No offending entries found" in caplog.text
def test_read_template_file_invalid_file(caplog, monkeypatch):
"""
GIVEN `read_template_file` is called
WHEN a non existent file is provided
THEN exit with an error of 1
"""
monkeypatch.setenv("CFN_TEMPLATE_FILE_LOCATION", "/tmp/x.yaml")
c = CcValidator()
with pytest.raises(SystemExit):
c.read_template_file()
assert "Template file does not exist" in caplog.text
def test_check_fail_pipeline_unset(monkeypatch, template_dir):
"""
GIVEN a valid template is passed in
WHEN `FAIL_PIPELINE_CFN` env var is `enabled` but `FailConformityPipeline` CFN parameter is not set
THEN return `True` (pipeline will fail when issues are found)
"""
monkeypatch.setenv("FAIL_PIPELINE_CFN", "enabled")
template_name = f"{template_dir}/insecure-s3-bucket.json"
with open(template_name, "r") as f:
cfn_contents = json.load(f)
c = CcValidator()
fail_pipeline = c._check_fail_pipeline(cfn_contents)
assert fail_pipeline is True
def test_fail_pipeline_disabled(monkeypatch):
"""
GIVEN `_fail_pipeline` is called
WHEN `FAIL_PIPELINE` is `disabled`
THEN return `False` (pipeline won't fail even if issues are found)
"""
monkeypatch.setenv("FAIL_PIPELINE", "disabled")
c = CcValidator()
# real template not required for testing
fail_pipeline = c._fail_pipeline("")
assert fail_pipeline is False
def test_run_validation_invalid_profile_id(monkeypatch):
"""
GIVEN `run_validation` is called
WHEN an invalid profile ID is provided
THEN exit with an error of 1
"""
monkeypatch.setenv("CC_PROFILE_ID", "x")
c = CcValidator()
template_str = c.read_template_file()
payload = c.generate_payload(template_str)
validation = c.run_validation(payload)
assert "errors" in validation
def test_check_fail_pipeline_disabled(monkeypatch, template_dir):
"""
GIVEN a valid template is passed in
WHEN `FAIL_PIPELINE_CFN` env var is `enabled` and `FailConformityPipeline` CFN parameter is `disabled`
THEN return `False` (pipeline won't fail even if issues are found)
"""
monkeypatch.setenv("FAIL_PIPELINE_CFN", "enabled")
template_name = f"{template_dir}/insecure-s3-bucket-disable-failure.json"
with open(template_name, "r") as f:
cfn_contents = json.load(f)
c = CcValidator()
fail_pipeline = c._check_fail_pipeline(cfn_contents)
assert fail_pipeline is False
def test_env_vars(set_env_vars):
"""
GIVEN `CcValidator` is instantiated
WHEN all suitable variables are passed
THEN don't raise an error
"""
CcValidator()
def test_run_insecure_template(caplog, monkeypatch, template_dir):
"""
GIVEN a valid template is passed in
WHEN the template has security issues
THEN exit with an error code of 1 and the failing rules
"""
insecure_file_path = f"{template_dir}/insecure-s3-bucket.json"
monkeypatch.setenv("CFN_TEMPLATE_FILE_LOCATION", insecure_file_path)
c = CcValidator()
with pytest.raises(SystemExit):
c.run()
assert "offending entries found" in caplog.text
def test_run_validation_valid_profile_id():
"""
GIVEN `run_validation` is called
WHEN a valid profile ID is provided
THEN return the payload provided by Conformity
"""
# ensure a valid profile ID is provided
if not os.environ.get("CC_PROFILE_ID"):
assert False is True
c = CcValidator()
template_str = c.read_template_file()
payload = c.generate_payload(template_str)
validation = c.run_validation(payload)
assert "data" in validation
def test_check_fail_pipeline_invalid(monkeypatch, template_dir):
"""
GIVEN a valid template is passed in
WHEN `FAIL_PIPELINE_CFN` env var is `enabled` and but `FailConformityPipeline` CFN parameter is set to something other than "disabled"
THEN return `True` (pipeline will fail when issues are found)
"""
monkeypatch.setenv("FAIL_PIPELINE_CFN", "enabled")
template_name = f"{template_dir}/insecure-s3-bucket-disable-failure.json"
with open(template_name, "r") as f:
cfn_contents = json.load(f)
cfn_contents["Parameters"]["FailConformityPipeline"] = "x"
c = CcValidator()
fail_pipeline = c._check_fail_pipeline(cfn_contents)
assert fail_pipeline is True
def test_fail_pipeline_template_files(monkeypatch,
disabled_failed_pipeline_templates):
"""
GIVEN `_fail_pipeline` is called
WHEN `FAIL_PIPELINE_CFN` env var is `enabled` and `FailConformityPipeline` CFN parameter is `disabled`
THEN return `False` (pipeline won't fail even if issues are found)
"""
# override default valid template file path
monkeypatch.setenv("CFN_TEMPLATE_FILE_LOCATION",
disabled_failed_pipeline_templates)
monkeypatch.setenv("FAIL_PIPELINE_CFN", "enabled")
with open(disabled_failed_pipeline_templates, "r") as f:
cfn_contents = f.read()
c = CcValidator()
fail_pipeline = c._fail_pipeline(cfn_contents)
assert fail_pipeline is False
def test_run_insecure_template_fail_pipeline_enabled_cfn_disabled(
caplog, monkeypatch, template_dir):
"""
GIVEN a valid template is passed in
WHEN the template has security issues and the `FAIL_PIPELINE_CFN` env var is `enabled` but the `FailConformityPipeline` CFN param is `disabled`
THEN exit with an error of 1 and the failing rules
"""
caplog.set_level(logging.INFO)
insecure_file_path = f"{template_dir}/insecure-s3-bucket-disable-failure.json"
monkeypatch.setenv("FAIL_PIPELINE_CFN", "disabled")
monkeypatch.setenv("CFN_TEMPLATE_FILE_LOCATION", insecure_file_path)
print(insecure_file_path)
c = CcValidator()
with pytest.raises(SystemExit):
c.run()
assert "offending entries found" in caplog.text
def test_run_insecure_template_fail_pipeline_disabled(caplog, monkeypatch,
template_dir):
"""
GIVEN a valid template is passed in
WHEN the template has security issues but the `FAIL_PIPELINE` environment variable is set to "disabled"
THEN exit with a code of 0 and the failing rules
"""
caplog.set_level(logging.INFO)
insecure_file_path = f"{template_dir}/insecure-s3-bucket.json"
monkeypatch.setenv("FAIL_PIPELINE", "disabled")
monkeypatch.setenv("CFN_TEMPLATE_FILE_LOCATION", insecure_file_path)
print(insecure_file_path)
c = CcValidator()
with pytest.raises(SystemExit):
c.run()
assert "Pipeline failure has been disabled" in caplog.text
def test_invalid_level(caplog, monkeypatch):
"""
GIVEN `CcValidator` is instantiated
WHEN an invalid rule severity level is provided
THEN exit with an error of 1
"""
monkeypatch.setenv("CC_RISK_LEVEL", "x")
with pytest.raises(SystemExit):
CcValidator()
assert "Unknown risk level. Please use one of LOW | MEDIUM | HIGH | VERY_HIGH | EXTREME" in caplog.text
def test_fail_pipeline_invalid_template_filename(caplog, monkeypatch,
tmp_path):
"""
GIVEN `_fail_pipeline` is called
WHEN `FAIL_PIPELINE_CFN` env var is `enabled` but an invalid filename is passed in
THEN exit with an error of 1
"""
d = tmp_path / "sub"
d.mkdir()
template_file_path = d / "x.txt"
# override default valid template file path
monkeypatch.setenv("CFN_TEMPLATE_FILE_LOCATION", str(template_file_path))
monkeypatch.setenv("FAIL_PIPELINE_CFN", "enabled")
c = CcValidator()
with pytest.raises(SystemExit):
c._fail_pipeline("")
assert "Unknown file extension for template" in caplog.text
def test_invalid_region(caplog, monkeypatch):
"""
GIVEN `CcValidator` is instantiated
WHEN an invalid CC region is provided
THEN exit with an error of 1
"""
monkeypatch.setenv("CC_REGION", "x")
with pytest.raises(SystemExit):
CcValidator()
assert 'Please ensure "CC_REGION" is set to a region which is supported by Conformity' in caplog.text
def test_missing_env_vars(caplog, monkeypatch):
"""
GIVEN `CcValidator` is instantiated
WHEN one or more env vars are not provided
THEN exit with an error of 1
"""
monkeypatch.delenv("CFN_TEMPLATE_FILE_LOCATION")
with pytest.raises(SystemExit):
CcValidator()
assert "Please ensure all environment variables are set" in caplog.text
def test_run_validation_invalid_api_key(caplog, monkeypatch):
"""
GIVEN `run_validation` is called
WHEN an invalid API key is provided
THEN exit with an error of 1
"""
monkeypatch.setenv("CC_API_KEY", "x")
c = CcValidator()
template_str = c.read_template_file()
payload = c.generate_payload(template_str)
with pytest.raises(SystemExit):
c.run_validation(payload)
assert "User is not authorized to access this resource with an explicit deny" in caplog.text
