def scan(self,
package: Package,
level: str,
exceptions: Exceptions = None) -> None:
"""Scan package looking for XML files."""
xml_files = [] # type: List[str]
globs = ["*.xml", "*.launch"] # type: List[str]
root = ""
for root, _, files in os.walk(package.path):
for glob in globs:
for f in fnmatch.filter(files, glob):
full_path = os.path.join(root, f)
xml_files.append(os.path.abspath(full_path))
xml_files = list(OrderedDict.fromkeys(xml_files))
print(" {} XML files found.".format(len(xml_files)))
if exceptions:
original_file_count = len(xml_files)
xml_files = exceptions.filter_file_exceptions_early(
package, xml_files)
if original_file_count > len(xml_files):
print(
" After filtering, {} XML files will be scanned.".format(
len(xml_files)))
package["xml"] = xml_files
def scan(self,
package: Package,
level: str,
exceptions: Exceptions = None) -> None:
"""Scan package looking for maven files."""
top_poms = [] # type: List[str]
all_poms = [] # type: List[str]
deepest_pom_level = 999999
for root, _, files in os.walk(package.path):
for f in fnmatch.filter(files, "pom.xml"):
full_path = os.path.join(root, f)
# Kind of an ugly hack, but it makes sure long paths don't
# mess up our depth tracking
if exceptions and not exceptions.filter_file_exceptions_early(
package, [full_path]):
continue
depth = full_path.count(os.sep)
if depth < deepest_pom_level:
deepest_pom_level = depth
top_poms = []
if depth == deepest_pom_level:
top_poms.append(full_path)
all_poms.append(full_path)
top_poms = list(OrderedDict.fromkeys(top_poms))
all_poms = list(OrderedDict.fromkeys(all_poms))
print(" {} Maven POM files found.".format(len(all_poms)))
print(" {} top-level Maven POM files found.".format(len(top_poms)))
package["all_poms"] = all_poms
package["top_poms"] = top_poms
def test_filter_file_exceptions_early():
"""
Test that filter_file_exceptions_early excludes files.
Expected result: Empty files list.
"""
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "early_exceptions.yaml")
)
package = Package("test", os.path.dirname(__file__))
files = [os.path.join(os.path.dirname(__file__), "unlikelystring")]
filtered_files = exceptions.filter_file_exceptions_early(package, files)
assert not filtered_files
def test_c_discovery_plugin_scan_exceptions():
"""Test that the C discovery plugin finds valid C source/header files."""
cdp = CDiscoveryPlugin()
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "exceptions.yaml"))
cdp.scan(package, "level", exceptions)
expected = [
"test.c",
"test.cpp",
"test.cc",
"test.cxx",
"test.h",
"test.hxx",
"test.hpp",
]
if cdp.file_command_exists():
expected += ["oddextensioncpp.source", "oddextensionc.source"]
# We have to add the path to each of the above
expected_fullpath = [
os.path.join(package.path, filename) for filename in expected
]
# Neat trick to verify that two unordered lists are the same
assert set(package["c_src"]) == set(expected_fullpath)
def test_package_exceptions():
"""
Test that package exceptions are found.
Expected result: no issues found
"""
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "package_exceptions.yaml")
)
package_exceptions = exceptions.get_exceptions(package)
assert len(package_exceptions["file"]) == 1
assert len(package_exceptions["message_regex"]) == 1
def test_exceptions_file_empty_string():
"""
Test for when a Exceptions is initialized with an empty string.
Expected result: ValueError is thrown
"""
with pytest.raises(ValueError):
Exceptions(os.path.join(""))
def test_exceptions_file_invalid_yaml():
"""
Test for when a Exceptions is initialized with an invalid yaml file.
Expected result: ValueError is thrown
"""
with pytest.raises(ValueError):
Exceptions(os.path.join(os.path.dirname(__file__), "bad.yaml"))
def test_exceptions_init_valid():
"""
Test that the Exceptions module initializes correctly.
Expected result: exceptions.exceptions is initialized.
"""
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), 'valid_exceptions.yaml'))
assert exceptions.exceptions
def test_exceptions_init_nonexistent():
"""
Test that the Exceptions module throws an IOError if a bad path is given.
Expected result: IOError thrown.
"""
with pytest.raises(IOError):
Exceptions(
os.path.join(os.path.dirname(__file__),
'nonexistent_exceptions.yaml'))
def test_filter_file_exceptions_early_dupes():
"""
Test that filter_file_exceptions_early excludes duplicated files.
I have no idea why one might have duplicate files, but might as well test it!
Expected result: Empty file list.
"""
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), 'early_exceptions.yaml'))
package = Package('test', os.path.dirname(__file__))
files = [
os.path.join(os.path.dirname(__file__), 'unlikelystring'),
os.path.join(os.path.dirname(__file__), 'unlikelystring')
]
filtered_files = exceptions.filter_file_exceptions_early(package, files)
assert not filtered_files
def test_maven_discovery_plugin_scan_with_exceptions():
"""Test that the Maven discovery plugin finds pom.xml files when exceptions are specified."""
mdp = MavenDiscoveryPlugin()
package = Package(
"single_package",
os.path.join(os.path.dirname(__file__), "single_package"))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "valid_exceptions.yaml"))
mdp.scan(package, "level", exceptions)
assert not package["top_poms"]
assert not package["all_poms"]
def test_yaml_discovery_plugin_scan_exceptions():
"""Test that the yaml discovery plugin properly respects exceptions."""
yamldp = YAMLDiscoveryPlugin()
package = Package('valid_package', os.path.join(os.path.dirname(__file__),
'valid_package'))
exceptions = Exceptions(os.path.join(os.path.dirname(__file__), 'exceptions.yaml'))
yamldp.scan(package, 'level', exceptions)
expected_src = ['test.yaml']
# We have to add the path to each of the above...yuck
expected_src_fullpath = [os.path.join(package.path, filename)
for filename in expected_src]
# Neat trick to verify that two unordered lists are the same
assert set(package['yaml']) == set(expected_src_fullpath)
def test_filter_issues():
"""
Test that issues are filtered based on regex exceptions.
Expected result: all but one non-excepted issue is filtered
"""
package = Package('valid_package',
os.path.join(os.path.dirname(__file__), 'valid_package'))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), 'valid_exceptions.yaml'))
filename = 'x.py'
line_number = '4'
tool = 'pylint'
issue_type = 'R0205(useless-object-inheritance)'
severity = '5'
message = "R0205: Class 'Example' inherits from object, can be safely removed from bases in python3"
tool_issue = Issue(filename, line_number, tool, issue_type, severity,
message, None)
issues = {}
issues['pylint'] = [tool_issue]
issues = exceptions.filter_issues(package, issues)
assert not issues['pylint']
def test_javascript_discovery_plugin_scan_exceptions():
"""Test that the JavaScript discovery plugin properly respects exceptions."""
discovery_plugin = JavaScriptDiscoveryPlugin()
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "exceptions.yaml"))
discovery_plugin.scan(package, "level", exceptions)
expected_src = ["test.js"]
# We have to add the path to each of the above...yuck
expected_src_fullpath = [
os.path.join(package.path, filename) for filename in expected_src
]
# Neat trick to verify that two unordered lists are the same
assert set(package["javascript_src"]) == set(expected_src_fullpath)
def test_perl_discovery_plugin_scan_exceptions():
"""Test that the perl discovery plugin properly respects exceptions."""
pldp = PerlDiscoveryPlugin()
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
exceptions = Exceptions(os.path.join(os.path.dirname(__file__), "exceptions.yaml"))
pldp.scan(package, "level", exceptions)
expected_src = ["test.pl", "oddextensionpl.source"]
# We have to add the path to each of the above...yuck
expected_src_fullpath = [
os.path.join(package.path, filename) for filename in expected_src
]
# Neat trick to verify that two unordered lists are the same
assert set(package["perl_src"]) == set(expected_src_fullpath)
def test_filter_issues_nolint_not_abs_path():
"""
Test that issues are not filtered based on NOLINT comment when not absolute path.
Expected result: one issue found
"""
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "valid_exceptions.yaml"))
filename = "valid_package/x.py"
line_number = "3"
tool = "pylint"
issue_type = "missing-docstring"
severity = "3"
message = "C0111: Missing module docstring"
tool_issue = Issue(filename, line_number, tool, issue_type, severity,
message, None)
issues = {}
issues["pylint"] = [tool_issue]
issues = exceptions.filter_issues(package, issues)
assert len(issues["pylint"]) == 1
def test_filter_issues_nolint_not_abs_path():
"""
Test that issues are not filtered based on NOLINT comment when not absolute path.
Expected result: one issue found
"""
package = Package('valid_package',
os.path.join(os.path.dirname(__file__), 'valid_package'))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), 'valid_exceptions.yaml'))
filename = 'valid_package/x.py'
line_number = '3'
tool = 'pylint'
issue_type = 'missing-docstring'
severity = '3'
message = 'C0111: Missing module docstring'
tool_issue = Issue(filename, line_number, tool, issue_type, severity,
message, None)
issues = {}
issues['pylint'] = [tool_issue]
issues = exceptions.filter_issues(package, issues)
assert len(issues['pylint']) == 1
def test_filter_issues_globs_wrong_file_pattern():
"""
Test that issues are filtered based on regex exceptions if it matches a glob.
Expected result: no issues are filtered and one issue is found.
"""
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "glob_exceptions.yaml"))
filename = "filename_does_not_match_glob_pattern.py"
line_number = "4"
tool = "pylint"
issue_type = "R0205(useless-object-inheritance)"
severity = "5"
message = "R0205: Class 'Example' inherits from object, can be safely removed from bases in python3"
tool_issue = Issue(filename, line_number, tool, issue_type, severity,
message, None)
issues = {}
issues["pylint"] = [tool_issue]
issues = exceptions.filter_issues(package, issues)
assert len(issues["pylint"]) == 1
def test_filter_issues_empty_exceptions():
"""
Test that issues are filtered when the exceptions file is empty.
Expected result: one issue is found.
"""
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "empty_exceptions.yaml"))
filename = "x.py"
line_number = "4"
tool = "pylint"
issue_type = "R0205(useless-object-inheritance)"
severity = "5"
message = "R0205: Class 'Example' inherits from object, can be safely removed from bases in python3"
tool_issue = Issue(filename, line_number, tool, issue_type, severity,
message, None)
issues = {}
issues["pylint"] = [tool_issue]
issues = exceptions.filter_issues(package, issues)
assert len(issues["pylint"]) == 1
def test_filter_issues_filename_abs_path():
"""
Test that issues are filtered based on regex exceptions with absolute path.
Expected result: no issues found
"""
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "valid_exceptions.yaml"))
filename = "/home/travis/build/x.py"
line_number = "4"
tool = "pylint"
issue_type = "R0205(useless-object-inheritance)"
severity = "5"
message = "R0205: Class 'Example' inherits from object, can be safely removed from bases in python3"
tool_issue = Issue(filename, line_number, tool, issue_type, severity,
message, None)
issues = {}
issues["pylint"] = [tool_issue]
issues = exceptions.filter_issues(package, issues)
assert not issues["pylint"]
def test_filter_issues_wildcard_exceptions():
"""
Test that issues are found even when exceptions with wildcards for regex are used.
Expected result: one issue found
"""
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "package_exceptions.yaml"))
filename = "valid_package/x.py"
line_number = "3"
tool = "pylint"
issue_type = "missing-docstring"
severity = "3"
message = "C0111: Missing module docstring"
tool_issue = Issue(filename, line_number, tool, issue_type, severity,
message, None)
issues = {}
issues["pylint"] = [tool_issue]
issues = exceptions.filter_issues(package, issues)
assert len(issues["pylint"]) == 1
def test_javascript_discovery_plugin_scan_exceptions():
"""Test that the JavaScript discovery plugin properly respects exceptions."""
discovery_plugin = JavaScriptDiscoveryPlugin()
package = Package('valid_package',
os.path.join(os.path.dirname(__file__), 'valid_package'))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), 'exceptions.yaml'))
discovery_plugin.scan(package, 'level', exceptions)
expected_src = ['test.js']
# We have to add the path to each of the above...yuck
expected_src_fullpath = [
os.path.join(package.path, filename) for filename in expected_src
]
# Neat trick to verify that two unordered lists are the same
print("package: {}".format(package['javascript_src']))
assert set(package['javascript_src']) == set(expected_src_fullpath)
def test_rst_discovery_plugin_scan_exceptions():
"""Test that the rst discovery plugin properly respects exceptions."""
discovery_plugin = RstDiscoveryPlugin()
package = Package("valid_package",
os.path.join(os.path.dirname(__file__), "valid_package"))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "exceptions.yaml"))
discovery_plugin.scan(package, "level", exceptions)
expected = ["test.rst"]
# We have to add the path to each of the above...yuck
expected_fullpath = [
os.path.join(package.path, filename) for filename in expected
]
# Neat trick to verify that two unordered lists are the same
print("package: {}".format(package["rst_src"]))
assert set(package["rst_src"]) == set(expected_fullpath)
def scan(self,
package: Package,
level: str,
exceptions: Exceptions = None) -> None:
"""Scan package looking for TeX files."""
tex_files = [] # type: List[str]
globs = ["*.tex", "*.bib"] # type: List[str]
file_cmd_exists = True # type: bool
if not DiscoveryPlugin.file_command_exists():
file_cmd_exists = False
root = "" # type: str
for root, _, files in os.walk(package.path):
for glob in globs:
for f in fnmatch.filter(files, glob):
full_path = os.path.join(root, f)
tex_files.append(os.path.abspath(full_path))
if file_cmd_exists:
for f in files:
full_path = os.path.join(root, f)
output = subprocess.check_output(["file", full_path],
universal_newlines=True)
if f.endswith(".sty") or f.endswith(".log") or f.endswith(
".cls"):
continue
# pylint: disable=unsupported-membership-test
if ("LaTeX document" in output
or "BibTeX text file" in output
or "LaTeX 2e document" in output):
# pylint: enable=unsupported-membership-test
tex_files.append(os.path.abspath(full_path))
tex_files = list(OrderedDict.fromkeys(tex_files))
print(" {} TeX files found.".format(len(tex_files)))
if exceptions:
original_file_count = len(tex_files) # type: int
tex_files = exceptions.filter_file_exceptions_early(
package, tex_files)
if original_file_count > len(tex_files):
print(
" After filtering, {} TeX files will be scanned.".format(
len(tex_files)))
package["tex"] = tex_files
def test_python_discovery_plugin_scan_exceptions():
"""Test that the python discovery plugin properly respects exceptions."""
pydp = PythonDiscoveryPlugin()
if not pydp.file_command_exists():
pytest.skip("File command does not exist. Skipping test that requires it.")
package = Package(
"valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
)
exceptions = Exceptions(os.path.join(os.path.dirname(__file__), "exceptions.yaml"))
pydp.scan(package, "level", exceptions)
expected_src = ["test.py", "oddextensionpy.source"]
# We have to add the path to each of the above...yuck
expected_src_fullpath = [
os.path.join(package.path, filename) for filename in expected_src
]
# Neat trick to verify that two unordered lists are the same
assert set(package["python_src"]) == set(expected_src_fullpath)
def test_html_discovery_plugin_scan_exceptions():
"""Test that the html discovery plugin properly respects exceptions."""
discovery_plugin = HTMLDiscoveryPlugin()
package = Package('valid_package',
os.path.join(os.path.dirname(__file__), 'valid_package'))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), 'exceptions.yaml'))
discovery_plugin.scan(package, 'level', exceptions)
expected_src = ['test.html']
if discovery_plugin.file_command_exists():
expected_src += ['oddextensionhtml.source']
# We have to add the path to each of the above...yuck
expected_src_fullpath = [
os.path.join(package.path, filename) for filename in expected_src
]
# Neat trick to verify that two unordered lists are the same
print("package: {}".format(package['html_src']))
assert set(package['html_src']) == set(expected_src_fullpath)
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None:
"""Scan package looking for python files."""
python_files = [] # type: List[str]
file_cmd_exists = True # type: bool
if not DiscoveryPlugin.file_command_exists():
file_cmd_exists = False
for root, _, files in os.walk(package.path):
for f in fnmatch.filter(files, "*.py"):
full_path = os.path.join(root, f)
python_files.append(os.path.abspath(full_path))
if file_cmd_exists:
for f in files:
full_path = os.path.join(root, f)
output = subprocess.check_output(
["file", full_path], universal_newlines=True
) # type: str
# pylint: disable=unsupported-membership-test
if (
"python script" in output or "Python script" in output
) and not f.endswith(".cfg"):
# pylint: enable=unsupported-membership-test
python_files.append(os.path.abspath(full_path))
python_files = list(OrderedDict.fromkeys(python_files))
print(" {} python files found.".format(len(python_files)))
if exceptions:
original_file_count = len(python_files)
python_files = exceptions.filter_file_exceptions_early(
package, python_files
)
if original_file_count > len(python_files):
print(
" After filtering, {} python files will be scanned.".format(
len(python_files)
)
)
package["python_src"] = python_files
def test_c_discovery_plugin_scan_exceptions():
"""Test that the C discovery plugin finds valid C source/header files."""
cdp = CDiscoveryPlugin()
package = Package('valid_package',
os.path.join(os.path.dirname(__file__), 'valid_package'))
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), 'exceptions.yaml'))
cdp.scan(package, 'level', exceptions)
expected = [
'test.c', 'test.cpp', 'test.cc', 'test.cxx', 'test.h', 'test.hxx',
'test.hpp'
]
if cdp.file_command_exists():
expected += ['oddextensioncpp.source', 'oddextensionc.source']
# We have to add the path to each of the above
expected_fullpath = [
os.path.join(package.path, filename) for filename in expected
]
# Neat trick to verify that two unordered lists are the same
assert set(package['c_src']) == set(expected_fullpath)
def scan(self,
package: Package,
level: str,
exceptions: Exceptions = None) -> None:
"""Scan package looking for C files."""
c_files = [] # type: List[str]
c_extensions = (".c", ".cc", ".cpp", ".cxx", ".h", ".hxx", ".hpp")
file_cmd_exists = True # type: bool
if not DiscoveryPlugin.file_command_exists():
file_cmd_exists = False
for root, _, files in os.walk(package.path):
for f in files:
if f.lower().endswith(c_extensions):
full_path = os.path.join(root, f)
c_files.append(os.path.abspath(full_path))
elif file_cmd_exists:
full_path = os.path.join(root, f)
output = subprocess.check_output(
["file", full_path],
universal_newlines=True) # type: str
if ("c source" in output.lower()
or "c program" in output.lower() or "c++ source"
in output.lower()) and not f.endswith(".cfg"):
c_files.append(os.path.abspath(full_path))
c_files = list(OrderedDict.fromkeys(c_files))
print(" {} C/C++ files found.".format(len(c_files)))
if exceptions:
original_file_count = len(c_files)
c_files = exceptions.filter_file_exceptions_early(package, c_files)
if original_file_count > len(c_files):
print(" After filtering, {} C/C++ files will be scanned.".
format(len(c_files)))
package["c_src"] = c_files
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None:
"""Scan package looking for Perl files."""
perl_files = [] # type: List[str]
file_cmd_exists = True # type: bool
if not DiscoveryPlugin.file_command_exists():
file_cmd_exists = False
for root, _, files in os.walk(package.path):
for f in fnmatch.filter(files, "*.pl"):
full_path = os.path.join(root, f)
perl_files.append(os.path.abspath(full_path))
if file_cmd_exists:
for f in files:
full_path = os.path.join(root, f)
output = subprocess.check_output(
["file", full_path], universal_newlines=True
) # type: str
if "perl script" in output.lower():
perl_files.append(os.path.abspath(full_path))
perl_files = list(OrderedDict.fromkeys(perl_files))
print(" {} Perl files found.".format(len(perl_files)))
if exceptions:
original_file_count = len(perl_files)
perl_files = exceptions.filter_file_exceptions_early(package, perl_files)
if original_file_count > len(perl_files):
print(
" After filtering, {} perl files will be scanned.".format(
len(perl_files)
)
)
package["perl_src"] = perl_files
