def test_jshint_tool_plugin_scan_valid():
"""Integration test: Make sure the jshint output hasn't changed."""
plugin = setup_jshint_tool_plugin()
if not plugin.command_exists("jshint"):
pytest.skip("Missing jshint executable.")
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
package["javascript_src"] = [
os.path.join(os.path.dirname(__file__), "valid_package",
"test_no_issues.js")
]
issues = plugin.scan(package, "level")
assert not issues
def test_bandit_tool_plugin_scan_empty_src():
"""
Test what happens when python_src is an empty list.
Expected result: issues is an empty list
"""
btp = setup_bandit_tool_plugin()
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
package["python_src"] = []
issues = btp.scan(package, "level")
assert len(issues) == 0
def test_make_tool_plugin_scan_oserror(mock_subprocess_check_output):
"""
Test what happens when an OSError is raised (usually means make doesn't exist).
Expected result: issues is None
"""
mock_subprocess_check_output.side_effect = OSError('mocked error')
mtp = setup_make_tool_plugin()
package = Package('valid_package',
os.path.join(os.path.dirname(__file__), 'valid_package'))
package['make_targets'] = 'make_targets'
issues = mtp.scan(package, 'level')
assert issues is None
def test_discovery_plugin_find_files_multiple():
"""Test that find_files will only walk the path once."""
dp = DiscoveryPlugin()
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
package._walked = True # pylint: disable=protected-access
expected_dict = {}
dp.find_files(package)
assert package._walked # pylint: disable=protected-access
assert package.files == expected_dict
def test_yaml_discovery_plugin_scan_valid():
"""Test that the YAML discovery plugin correctly identifies YAML files."""
ydp = YAMLDiscoveryPlugin()
package = Package('valid_package',
os.path.join(os.path.dirname(__file__), 'valid_package'))
ydp.scan(package, 'level')
expected = ['test.yaml']
# We have to add the path to each of the above...yuck
expected_fullpath = [
os.path.join(package.path, filename) for filename in expected
]
# Neat trick to verify that two unordered lists are the same
assert set(package['yaml']) == set(expected_fullpath)
def test_black_tool_plugin_scan_valid():
"""Integration test: Make sure the black output hasn't changed."""
btp = setup_black_tool_plugin()
if not btp.command_exists("black"):
pytest.skip("Can't find black, unable to test black plugin.")
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
package["python_src"] = [
os.path.join(os.path.dirname(__file__), "valid_package", "format_errors.py")
]
issues = btp.scan(package, "level")
assert len(issues) == 1
def test_docformatter_tool_plugin_scan_valid():
"""Integration test: Make sure the docformatter output hasn't changed."""
dtp = setup_docformatter_tool_plugin()
if not dtp.command_exists("docformatter"):
pytest.skip(
"Can't find docformatter, unable to test docformatter plugin")
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
package["python_src"] = [
os.path.join(os.path.dirname(__file__), "valid_package", "wrong.py")
]
issues = dtp.scan(package, "level")
assert len(issues) == 1
def test_make_tool_plugin_parse_overloaded_virtual():
"""Verify that we can parse the output of make with overloaded-virtual."""
mtp = setup_make_tool_plugin()
package = Package("valid_package", "/home/user/valid_package")
output = (
"/home/user/valid_package/hello.c:7:3: error: overloaded-virtual: \n"
"/home/user/valid_package/hello.c:7:3: error: second line")
issues = mtp.parse_output(package, output)
assert len(issues) == 1
assert issues[0].filename == "/home/user/valid_package/hello.c"
assert issues[0].line_number == "7"
assert issues[0].severity == "5"
assert issues[0].message == "overloaded-virtual: second line"
def test_xmllint_tool_plugin_scan_valid():
"""Integration test: Make sure the xmllint output hasn't changed."""
xltp = setup_xmllint_tool_plugin()
if not xltp.command_exists('xmllint'):
pytest.skip('Missing xmllint executable.')
package = Package('valid_package',
os.path.join(os.path.dirname(__file__), 'valid_package'))
package['xml'] = [
os.path.join(os.path.dirname(__file__), 'valid_package',
'premature_end.xml')
]
issues = xltp.scan(package, 'level')
assert len(issues) == 2
def test_lacheck_tool_plugin_scan_valid():
"""Integration test: Make sure the lacheck output hasn't changed."""
ltp = setup_lacheck_tool_plugin()
if not ltp.command_exists("lacheck"):
pytest.skip("Missing lacheck executable.")
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
package["tex"] = [
os.path.join(os.path.dirname(__file__), "valid_package", "test.tex")
]
issues = ltp.scan(package, "level")
# We expect to have space before punctuation mark warning.
assert len(issues) == 1
def test_clang_format_tool_plugin_scan_oserror_raise(mock_open):
"""
Test what happens when OSError is raised (usually means clang-format configuration is missing).
Expected result: issues is None
"""
mock_open.side_effect = OSError("mocked error")
cftp = setup_clang_format_tool_plugin()
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
package["make_targets"] = []
package["headers"] = []
issues = cftp.scan(package, "level")
assert issues is None
def test_spotbugs_tool_plugin_scan_calledprocesserror(mock_subprocess_check_output):
"""
Test what happens when a CalledProcessError is raised (usually means maven hit an error).
Expected result: issues is None
"""
mock_subprocess_check_output.side_effect = subprocess.CalledProcessError(1, '', output='error')
sbtp = setup_spotbugs_tool_plugin()
package = Package('valid_package', os.path.join(os.path.dirname(__file__),
'valid_package'))
package['top_poms'] = [os.path.join(package.path, 'pom.xml')]
package['all_poms'] = [os.path.join(package.path, 'pom.xml')]
issues = sbtp.scan(package, 'level')
assert issues is None
def test_cppcheck_tool_plugin_scan_no_plugin_context():
"""Test that issues are None when no plugin context is provided."""
cctp = setup_cppcheck_tool_plugin(use_plugin_context=False)
if not cctp.command_exists("cppcheck"):
pytest.skip("Can't find cppcheck, unable to test cppcheck plugin")
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
package["make_targets"] = []
package["make_targets"].append({})
package["make_targets"][0]["src"] = []
package["headers"] = []
issues = cctp.scan(package, "level")
assert not issues
def test_clang_tidy_tool_plugin_scan_valid():
"""Integration test: Make sure the clang_tidy output hasn't changed."""
if sys.platform == "win32":
pytest.skip(
"Running CMake on GitHub Windows runner is failing. Skipping test."
)
cttp = setup_clang_tidy_tool_plugin()
if not cttp.command_exists("cmake"):
pytest.skip("Can't find CMake, unable to test clang_tidy plugin")
elif not cttp.command_exists("clang-tidy"):
pytest.skip("Can't find clang-tidy, unable to test clang_tidy plugin")
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
# Need to run CMake
with TemporaryDirectory() as bin_dir:
try:
subprocess.check_output(
[
"cmake",
os.path.join(os.path.dirname(__file__), "valid_package"),
"-DCMAKE_BUILD_TYPE=RelWithDebInfo",
"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
"-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=" + bin_dir,
],
universal_newlines=True,
cwd=bin_dir,
)
except subprocess.CalledProcessError as ex:
print(f"Problem running CMake! Returncode = {str(ex.returncode)}")
print(f"{ex.output}")
pytest.fail("Failed running CMake")
package["make_targets"] = []
package["make_targets"].append({})
package["make_targets"][0]["src"] = [
os.path.join(os.path.dirname(__file__), "valid_package", "test.c")
]
package["bin_dir"] = bin_dir
package["src_dir"] = os.path.join(os.path.dirname(__file__),
"valid_package")
issues = cttp.scan(package, "level")
assert len(issues) == 1
assert issues[0].filename == os.path.join(os.path.dirname(__file__),
"valid_package", "test.c")
assert issues[0].line_number == "6"
assert issues[0].tool == "clang-tidy"
assert issues[0].issue_type == "warning/clang-analyzer-deadcode.DeadStores"
assert issues[0].severity == "3"
assert issues[0].message == "Value stored to 'si' is never read"
def test_yamllint_tool_plugin_scan_valid():
"""Integration test: Make sure the yamllint output hasn't changed."""
yltp = setup_yamllint_tool_plugin()
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
package["yaml"] = [
os.path.join(os.path.dirname(__file__), "valid_package", "document-start.yaml")
]
issues = yltp.scan(package, "level")
if sys.platform == "win32":
assert len(issues) == 2 # additional error about newline character
else:
assert len(issues) == 1
def test_fortify_parse_nocontext_audit(fortify_tool_plugin):
"""Test that we can parse an fvdl file with a vulnerability with empty context."""
package = Package('test', os.path.dirname(__file__))
tree = etree.parse(os.path.join(os.path.dirname(__file__),
'nocontext_audit.fvdl'))
root = tree.getroot()
issues = fortify_tool_plugin.parse_output(root, package)
assert len(issues) == 1
assert issues[0].filename
assert issues[0].line_number == '1'
assert issues[0].tool == 'fortify'
assert issues[0].issue_type == 'configuration'
assert issues[0].severity == '2'
assert issues[0].message
def test_spotbugs_tool_plugin_scan_oserror(mock_subprocess_check_output):
"""
Test what happens when an OSError is raised (usually means maven doesn't exist - unlikely).
Expected result: issues is None
"""
mock_subprocess_check_output.side_effect = OSError("mocked error")
sbtp = setup_spotbugs_tool_plugin()
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
package["top_poms"] = [os.path.join(package.path, "pom.xml")]
package["all_poms"] = [os.path.join(package.path, "pom.xml")]
issues = sbtp.scan(package, "level")
assert issues is None
def test_perl_discovery_plugin_scan_valid():
"""Test that the Perl discovery plugin finds valid perl files."""
pldp = PerlDiscoveryPlugin()
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
pldp.scan(package, "level", None)
expected = ["test.pl", os.path.join("ignore_this", "ignoreme.pl")]
if pldp.file_command_exists():
expected += ["oddextensionpl.source"]
# We have to add the path to each of the above...yuck
expected_fullpath = [os.path.join(package.path, filename) for filename in expected]
# Neat trick to verify that two unordered lists are the same
assert set(package["perl_src"]) == set(expected_fullpath)
def test_chktex_tool_plugin_scan_valid():
"""Integration test: Make sure the chktex output hasn't changed."""
cttp = setup_chktex_tool_plugin()
if not cttp.command_exists("chktex"):
pytest.skip("Missing chktex executable.")
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
package["tex"] = [
os.path.join(os.path.dirname(__file__), "valid_package", "test.tex")
]
issues = cttp.scan(package, "level")
# We expect to have length of dash warning.
assert len(issues) == 1
def test_cmake_discovery_plugin_scan_calledprocesserror(
mock_subprocess_check_output):
"""
Test what happens when a CalledProcessError is raised (usually means yamllint hit an error).
Expected result: issues is None
"""
mock_subprocess_check_output.side_effect = subprocess.CalledProcessError(
1, "", output="mocked error")
cmdp = setup_cmake_discovery_plugin()
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
cmdp.scan(package, "level")
assert not package["make_targets"]
def test_pyflakes_tool_plugin_scan_oserror(mock_subprocess_check_output):
"""
Test what happens when an OSError is raised (usually means pyflakes doesn't exist).
Expected result: issues is None
"""
mock_subprocess_check_output.side_effect = OSError('mocked error')
pltp = setup_pyflakes_tool_plugin()
package = Package('valid_package', os.path.join(os.path.dirname(__file__),
'valid_package'))
package['python_src'] = [os.path.join(os.path.dirname(__file__),
'valid_package', 'pyflakes_test.py')]
issues = pltp.scan(package, 'level')
assert issues is None
def test_lizard_tool_plugin_scan_oserror(mock_subprocess_check_output):
"""
Test what happens when an OSError is raised (usually means lizard doesn't exist).
Expected result: issues is None
"""
mock_subprocess_check_output.side_effect = OSError("mocked error")
ltp = setup_lizard_tool_plugin()
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
package["src_dir"] = os.path.join(os.path.dirname(__file__),
"valid_package")
issues = ltp.scan(package, "level")
assert issues is None
def test_fortify_parse_class_audit(fortify_tool_plugin):
"""Test that we can parse a stripped-down sample fvdl file with a class vulnerability."""
package = Package('test', os.path.dirname(__file__))
tree = etree.parse(os.path.join(os.path.dirname(__file__),
'class_audit.fvdl'))
root = tree.getroot()
issues = fortify_tool_plugin.parse_output(root, package)
assert len(issues) == 1
assert issues[0].filename
assert issues[0].line_number == '542'
assert issues[0].tool == 'fortify'
assert issues[0].issue_type == 'structural'
assert issues[0].severity == '3'
assert issues[0].message
def test_maven_discovery_plugin_scan_multilevel():
"""Test that the Maven discovery plugin finds pom.xml files at multiple depths."""
mdp = MavenDiscoveryPlugin()
package = Package('multi_package', os.path.join(os.path.dirname(__file__),
'multi_package'))
mdp.scan(package, 'level')
expected_top = [os.path.join('a', 'pom.xml'), os.path.join('b', 'pom.xml')]
expected_all = expected_top + [os.path.join('a', 'c', 'pom.xml')]
expected_top_fullpath = [os.path.join(package.path, filename)
for filename in expected_top]
expected_all_fullpath = [os.path.join(package.path, filename)
for filename in expected_all]
assert set(package['top_poms']) == set(expected_top_fullpath)
assert set(package['all_poms']) == set(expected_all_fullpath)
def test_stylelint_tool_plugin_scan_valid_with_issues():
"""Integration test: Make sure the stylelint output hasn't changed."""
plugin = setup_stylelint_tool_plugin()
package = Package('valid_package',
os.path.join(os.path.dirname(__file__), 'valid_package'))
package['html_src'] = [
os.path.join(os.path.dirname(__file__), 'valid_package', 'test.html')
]
package['css_src'] = [
os.path.join(os.path.dirname(__file__), 'valid_package', 'test.css')
]
issues = plugin.scan(package, 'level')
# We expect to have declaration-colon-after-space and block-opening-brace-space-before errors.
assert len(issues) == 3
def test_markdown_plugin_scan_valid():
"""Test that the markdown discovery plugin finds valid markdown source and bib files."""
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
discovery_plugin = MarkdownDiscoveryPlugin()
discovery_plugin.scan(package, "level")
expected = ["test.md", os.path.join("ignore_this", "ignoreme.md")]
# if discovery_plugin.file_command_exists():
# expected += ['oddextensionmarkdown.source']
# We have to add the path to each of the above...yuck
expected_fullpath = [os.path.join(package.path, filename) for filename in expected]
# Neat trick to verify that two unordered lists are the same
assert set(package["md_src"]) == set(expected_fullpath)
def test_cppcheck_tool_plugin_scan_no_files():
"""Check what happens if the plugin isn't passed any files."""
cctp = setup_cppcheck_tool_plugin()
if not cctp.command_exists("cppcheck"):
pytest.skip("Can't find cppcheck, unable to test cppcheck plugin")
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
package["make_targets"] = []
package["make_targets"].append({})
package["make_targets"][0]["src"] = []
package["headers"] = []
issues = cctp.scan(package, "level")
assert not issues
def test_perl_discovery_plugin_scan_valid():
"""Test that the Perl discovery plugin finds valid perl files."""
pldp = PerlDiscoveryPlugin()
package = Package('valid_package', os.path.join(os.path.dirname(__file__),
'valid_package'))
pldp.scan(package, 'level', None)
expected = ['test.pl']
if pldp.file_command_exists():
expected += ['oddextensionpl.source']
# We have to add the path to each of the above...yuck
expected_fullpath = [os.path.join(package.path, filename)
for filename in expected]
# Neat trick to verify that two unordered lists are the same
assert set(package['perl_src']) == set(expected_fullpath)
def test_cpplint_tool_plugin_scan_valid():
"""Integration test: Make sure the cpplint output hasn't changed."""
ctp = setup_cpplint_tool_plugin()
if not ctp.command_exists("cmake"):
pytest.skip("Can't find CMake, unable to test cpplint plugin")
elif not ctp.command_exists("cpplint"):
pytest.skip("Can't find cpplint, unable to test cpplint plugin")
elif sys.platform == "win32":
pytest.skip("Don't know how to run cpplint on Windows.")
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
# Need to actually run CMake to generate compile_commands.json
with TemporaryDirectory() as bin_dir:
try:
subprocess.check_output(
[
"cmake",
os.path.join(os.path.dirname(__file__), "valid_package"),
"-DCMAKE_BUILD_TYPE=RelWithDebInfo",
"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
"-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=" + bin_dir,
],
universal_newlines=True,
cwd=bin_dir,
)
except subprocess.CalledProcessError as ex:
print("Problem running CMake! Returncode = {}".format(str(ex.returncode)))
print("{}".format(ex.output))
pytest.fail("Failed running CMake")
package["make_targets"] = []
package["make_targets"].append({})
package["make_targets"][0]["src"] = [
os.path.join(os.path.dirname(__file__), "valid_package", "test.c")
]
package["headers"] = []
package["cpplint"] = "cpplint"
issues = ctp.scan(package, "level")
print("Line: {}".format(issues[2].message))
assert len(issues) == 4
assert issues[2].filename == os.path.join(
os.path.dirname(__file__), "valid_package", "test.c"
)
assert issues[2].line_number == "6"
assert issues[2].tool == "cpplint"
assert issues[2].issue_type == "whitespace/line_length"
assert issues[2].severity == "2"
assert issues[2].message == " Lines should be <= 80 characters long "
def test_clang_format_tool_plugin_scan_oserror_no_raise(mock_open):
"""
Test what happens when OSError is raised (usually means clang-format configuration is missing).
Expected result: issues is empty
"""
mock_open.side_effect = OSError('mocked error')
cftp = setup_clang_format_tool_plugin_non_default()
package = Package('valid_package', os.path.join(os.path.dirname(__file__),
'valid_package'))
package['make_targets'] = []
package['headers'] = []
issues = cftp.scan(package, 'level')
assert not issues
