def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for XML files.""" xml_files = [] # type: List[str] globs = ["*.xml", "*.launch"] # type: List[str] root = "" for root, _, files in os.walk(package.path): for glob in globs: for f in fnmatch.filter(files, glob): full_path = os.path.join(root, f) xml_files.append(os.path.abspath(full_path)) xml_files = list(OrderedDict.fromkeys(xml_files)) print(" {} XML files found.".format(len(xml_files))) if exceptions: original_file_count = len(xml_files) xml_files = exceptions.filter_file_exceptions_early( package, xml_files) if original_file_count > len(xml_files): print( " After filtering, {} XML files will be scanned.".format( len(xml_files))) package["xml"] = xml_files
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for maven files.""" top_poms = [] # type: List[str] all_poms = [] # type: List[str] deepest_pom_level = 999999 for root, _, files in os.walk(package.path): for f in fnmatch.filter(files, "pom.xml"): full_path = os.path.join(root, f) # Kind of an ugly hack, but it makes sure long paths don't # mess up our depth tracking if exceptions and not exceptions.filter_file_exceptions_early( package, [full_path]): continue depth = full_path.count(os.sep) if depth < deepest_pom_level: deepest_pom_level = depth top_poms = [] if depth == deepest_pom_level: top_poms.append(full_path) all_poms.append(full_path) top_poms = list(OrderedDict.fromkeys(top_poms)) all_poms = list(OrderedDict.fromkeys(all_poms)) print(" {} Maven POM files found.".format(len(all_poms))) print(" {} top-level Maven POM files found.".format(len(top_poms))) package["all_poms"] = all_poms package["top_poms"] = top_poms
def test_filter_file_exceptions_early(): """ Test that filter_file_exceptions_early excludes files. Expected result: Empty files list. """ exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "early_exceptions.yaml") ) package = Package("test", os.path.dirname(__file__)) files = [os.path.join(os.path.dirname(__file__), "unlikelystring")] filtered_files = exceptions.filter_file_exceptions_early(package, files) assert not filtered_files
def test_c_discovery_plugin_scan_exceptions(): """Test that the C discovery plugin finds valid C source/header files.""" cdp = CDiscoveryPlugin() package = Package("valid_package", os.path.join(os.path.dirname(__file__), "valid_package")) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "exceptions.yaml")) cdp.scan(package, "level", exceptions) expected = [ "test.c", "test.cpp", "test.cc", "test.cxx", "test.h", "test.hxx", "test.hpp", ] if cdp.file_command_exists(): expected += ["oddextensioncpp.source", "oddextensionc.source"] # We have to add the path to each of the above expected_fullpath = [ os.path.join(package.path, filename) for filename in expected ] # Neat trick to verify that two unordered lists are the same assert set(package["c_src"]) == set(expected_fullpath)
def test_package_exceptions(): """ Test that package exceptions are found. Expected result: no issues found """ package = Package( "valid_package", os.path.join(os.path.dirname(__file__), "valid_package") ) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "package_exceptions.yaml") ) package_exceptions = exceptions.get_exceptions(package) assert len(package_exceptions["file"]) == 1 assert len(package_exceptions["message_regex"]) == 1
def test_exceptions_file_empty_string(): """ Test for when a Exceptions is initialized with an empty string. Expected result: ValueError is thrown """ with pytest.raises(ValueError): Exceptions(os.path.join(""))
def test_exceptions_file_invalid_yaml(): """ Test for when a Exceptions is initialized with an invalid yaml file. Expected result: ValueError is thrown """ with pytest.raises(ValueError): Exceptions(os.path.join(os.path.dirname(__file__), "bad.yaml"))
def test_exceptions_init_valid(): """ Test that the Exceptions module initializes correctly. Expected result: exceptions.exceptions is initialized. """ exceptions = Exceptions( os.path.join(os.path.dirname(__file__), 'valid_exceptions.yaml')) assert exceptions.exceptions
def test_exceptions_init_nonexistent(): """ Test that the Exceptions module throws an IOError if a bad path is given. Expected result: IOError thrown. """ with pytest.raises(IOError): Exceptions( os.path.join(os.path.dirname(__file__), 'nonexistent_exceptions.yaml'))
def test_filter_file_exceptions_early_dupes(): """ Test that filter_file_exceptions_early excludes duplicated files. I have no idea why one might have duplicate files, but might as well test it! Expected result: Empty file list. """ exceptions = Exceptions( os.path.join(os.path.dirname(__file__), 'early_exceptions.yaml')) package = Package('test', os.path.dirname(__file__)) files = [ os.path.join(os.path.dirname(__file__), 'unlikelystring'), os.path.join(os.path.dirname(__file__), 'unlikelystring') ] filtered_files = exceptions.filter_file_exceptions_early(package, files) assert not filtered_files
def test_maven_discovery_plugin_scan_with_exceptions(): """Test that the Maven discovery plugin finds pom.xml files when exceptions are specified.""" mdp = MavenDiscoveryPlugin() package = Package( "single_package", os.path.join(os.path.dirname(__file__), "single_package")) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "valid_exceptions.yaml")) mdp.scan(package, "level", exceptions) assert not package["top_poms"] assert not package["all_poms"]
def test_yaml_discovery_plugin_scan_exceptions(): """Test that the yaml discovery plugin properly respects exceptions.""" yamldp = YAMLDiscoveryPlugin() package = Package('valid_package', os.path.join(os.path.dirname(__file__), 'valid_package')) exceptions = Exceptions(os.path.join(os.path.dirname(__file__), 'exceptions.yaml')) yamldp.scan(package, 'level', exceptions) expected_src = ['test.yaml'] # We have to add the path to each of the above...yuck expected_src_fullpath = [os.path.join(package.path, filename) for filename in expected_src] # Neat trick to verify that two unordered lists are the same assert set(package['yaml']) == set(expected_src_fullpath)
def test_filter_issues(): """ Test that issues are filtered based on regex exceptions. Expected result: all but one non-excepted issue is filtered """ package = Package('valid_package', os.path.join(os.path.dirname(__file__), 'valid_package')) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), 'valid_exceptions.yaml')) filename = 'x.py' line_number = '4' tool = 'pylint' issue_type = 'R0205(useless-object-inheritance)' severity = '5' message = "R0205: Class 'Example' inherits from object, can be safely removed from bases in python3" tool_issue = Issue(filename, line_number, tool, issue_type, severity, message, None) issues = {} issues['pylint'] = [tool_issue] issues = exceptions.filter_issues(package, issues) assert not issues['pylint']
def test_javascript_discovery_plugin_scan_exceptions(): """Test that the JavaScript discovery plugin properly respects exceptions.""" discovery_plugin = JavaScriptDiscoveryPlugin() package = Package("valid_package", os.path.join(os.path.dirname(__file__), "valid_package")) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "exceptions.yaml")) discovery_plugin.scan(package, "level", exceptions) expected_src = ["test.js"] # We have to add the path to each of the above...yuck expected_src_fullpath = [ os.path.join(package.path, filename) for filename in expected_src ] # Neat trick to verify that two unordered lists are the same assert set(package["javascript_src"]) == set(expected_src_fullpath)
def test_perl_discovery_plugin_scan_exceptions(): """Test that the perl discovery plugin properly respects exceptions.""" pldp = PerlDiscoveryPlugin() package = Package( "valid_package", os.path.join(os.path.dirname(__file__), "valid_package") ) exceptions = Exceptions(os.path.join(os.path.dirname(__file__), "exceptions.yaml")) pldp.scan(package, "level", exceptions) expected_src = ["test.pl", "oddextensionpl.source"] # We have to add the path to each of the above...yuck expected_src_fullpath = [ os.path.join(package.path, filename) for filename in expected_src ] # Neat trick to verify that two unordered lists are the same assert set(package["perl_src"]) == set(expected_src_fullpath)
def test_filter_issues_nolint_not_abs_path(): """ Test that issues are not filtered based on NOLINT comment when not absolute path. Expected result: one issue found """ package = Package("valid_package", os.path.join(os.path.dirname(__file__), "valid_package")) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "valid_exceptions.yaml")) filename = "valid_package/x.py" line_number = "3" tool = "pylint" issue_type = "missing-docstring" severity = "3" message = "C0111: Missing module docstring" tool_issue = Issue(filename, line_number, tool, issue_type, severity, message, None) issues = {} issues["pylint"] = [tool_issue] issues = exceptions.filter_issues(package, issues) assert len(issues["pylint"]) == 1
def test_filter_issues_nolint_not_abs_path(): """ Test that issues are not filtered based on NOLINT comment when not absolute path. Expected result: one issue found """ package = Package('valid_package', os.path.join(os.path.dirname(__file__), 'valid_package')) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), 'valid_exceptions.yaml')) filename = 'valid_package/x.py' line_number = '3' tool = 'pylint' issue_type = 'missing-docstring' severity = '3' message = 'C0111: Missing module docstring' tool_issue = Issue(filename, line_number, tool, issue_type, severity, message, None) issues = {} issues['pylint'] = [tool_issue] issues = exceptions.filter_issues(package, issues) assert len(issues['pylint']) == 1
def test_filter_issues_globs_wrong_file_pattern(): """ Test that issues are filtered based on regex exceptions if it matches a glob. Expected result: no issues are filtered and one issue is found. """ package = Package("valid_package", os.path.join(os.path.dirname(__file__), "valid_package")) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "glob_exceptions.yaml")) filename = "filename_does_not_match_glob_pattern.py" line_number = "4" tool = "pylint" issue_type = "R0205(useless-object-inheritance)" severity = "5" message = "R0205: Class 'Example' inherits from object, can be safely removed from bases in python3" tool_issue = Issue(filename, line_number, tool, issue_type, severity, message, None) issues = {} issues["pylint"] = [tool_issue] issues = exceptions.filter_issues(package, issues) assert len(issues["pylint"]) == 1
def test_filter_issues_empty_exceptions(): """ Test that issues are filtered when the exceptions file is empty. Expected result: one issue is found. """ package = Package("valid_package", os.path.join(os.path.dirname(__file__), "valid_package")) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "empty_exceptions.yaml")) filename = "x.py" line_number = "4" tool = "pylint" issue_type = "R0205(useless-object-inheritance)" severity = "5" message = "R0205: Class 'Example' inherits from object, can be safely removed from bases in python3" tool_issue = Issue(filename, line_number, tool, issue_type, severity, message, None) issues = {} issues["pylint"] = [tool_issue] issues = exceptions.filter_issues(package, issues) assert len(issues["pylint"]) == 1
def test_filter_issues_filename_abs_path(): """ Test that issues are filtered based on regex exceptions with absolute path. Expected result: no issues found """ package = Package("valid_package", os.path.join(os.path.dirname(__file__), "valid_package")) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "valid_exceptions.yaml")) filename = "/home/travis/build/x.py" line_number = "4" tool = "pylint" issue_type = "R0205(useless-object-inheritance)" severity = "5" message = "R0205: Class 'Example' inherits from object, can be safely removed from bases in python3" tool_issue = Issue(filename, line_number, tool, issue_type, severity, message, None) issues = {} issues["pylint"] = [tool_issue] issues = exceptions.filter_issues(package, issues) assert not issues["pylint"]
def test_filter_issues_wildcard_exceptions(): """ Test that issues are found even when exceptions with wildcards for regex are used. Expected result: one issue found """ package = Package("valid_package", os.path.join(os.path.dirname(__file__), "valid_package")) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "package_exceptions.yaml")) filename = "valid_package/x.py" line_number = "3" tool = "pylint" issue_type = "missing-docstring" severity = "3" message = "C0111: Missing module docstring" tool_issue = Issue(filename, line_number, tool, issue_type, severity, message, None) issues = {} issues["pylint"] = [tool_issue] issues = exceptions.filter_issues(package, issues) assert len(issues["pylint"]) == 1
def test_javascript_discovery_plugin_scan_exceptions(): """Test that the JavaScript discovery plugin properly respects exceptions.""" discovery_plugin = JavaScriptDiscoveryPlugin() package = Package('valid_package', os.path.join(os.path.dirname(__file__), 'valid_package')) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), 'exceptions.yaml')) discovery_plugin.scan(package, 'level', exceptions) expected_src = ['test.js'] # We have to add the path to each of the above...yuck expected_src_fullpath = [ os.path.join(package.path, filename) for filename in expected_src ] # Neat trick to verify that two unordered lists are the same print("package: {}".format(package['javascript_src'])) assert set(package['javascript_src']) == set(expected_src_fullpath)
def test_rst_discovery_plugin_scan_exceptions(): """Test that the rst discovery plugin properly respects exceptions.""" discovery_plugin = RstDiscoveryPlugin() package = Package("valid_package", os.path.join(os.path.dirname(__file__), "valid_package")) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "exceptions.yaml")) discovery_plugin.scan(package, "level", exceptions) expected = ["test.rst"] # We have to add the path to each of the above...yuck expected_fullpath = [ os.path.join(package.path, filename) for filename in expected ] # Neat trick to verify that two unordered lists are the same print("package: {}".format(package["rst_src"])) assert set(package["rst_src"]) == set(expected_fullpath)
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for TeX files.""" tex_files = [] # type: List[str] globs = ["*.tex", "*.bib"] # type: List[str] file_cmd_exists = True # type: bool if not DiscoveryPlugin.file_command_exists(): file_cmd_exists = False root = "" # type: str for root, _, files in os.walk(package.path): for glob in globs: for f in fnmatch.filter(files, glob): full_path = os.path.join(root, f) tex_files.append(os.path.abspath(full_path)) if file_cmd_exists: for f in files: full_path = os.path.join(root, f) output = subprocess.check_output(["file", full_path], universal_newlines=True) if f.endswith(".sty") or f.endswith(".log") or f.endswith( ".cls"): continue # pylint: disable=unsupported-membership-test if ("LaTeX document" in output or "BibTeX text file" in output or "LaTeX 2e document" in output): # pylint: enable=unsupported-membership-test tex_files.append(os.path.abspath(full_path)) tex_files = list(OrderedDict.fromkeys(tex_files)) print(" {} TeX files found.".format(len(tex_files))) if exceptions: original_file_count = len(tex_files) # type: int tex_files = exceptions.filter_file_exceptions_early( package, tex_files) if original_file_count > len(tex_files): print( " After filtering, {} TeX files will be scanned.".format( len(tex_files))) package["tex"] = tex_files
def test_python_discovery_plugin_scan_exceptions(): """Test that the python discovery plugin properly respects exceptions.""" pydp = PythonDiscoveryPlugin() if not pydp.file_command_exists(): pytest.skip("File command does not exist. Skipping test that requires it.") package = Package( "valid_package", os.path.join(os.path.dirname(__file__), "valid_package") ) exceptions = Exceptions(os.path.join(os.path.dirname(__file__), "exceptions.yaml")) pydp.scan(package, "level", exceptions) expected_src = ["test.py", "oddextensionpy.source"] # We have to add the path to each of the above...yuck expected_src_fullpath = [ os.path.join(package.path, filename) for filename in expected_src ] # Neat trick to verify that two unordered lists are the same assert set(package["python_src"]) == set(expected_src_fullpath)
def test_html_discovery_plugin_scan_exceptions(): """Test that the html discovery plugin properly respects exceptions.""" discovery_plugin = HTMLDiscoveryPlugin() package = Package('valid_package', os.path.join(os.path.dirname(__file__), 'valid_package')) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), 'exceptions.yaml')) discovery_plugin.scan(package, 'level', exceptions) expected_src = ['test.html'] if discovery_plugin.file_command_exists(): expected_src += ['oddextensionhtml.source'] # We have to add the path to each of the above...yuck expected_src_fullpath = [ os.path.join(package.path, filename) for filename in expected_src ] # Neat trick to verify that two unordered lists are the same print("package: {}".format(package['html_src'])) assert set(package['html_src']) == set(expected_src_fullpath)
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for python files.""" python_files = [] # type: List[str] file_cmd_exists = True # type: bool if not DiscoveryPlugin.file_command_exists(): file_cmd_exists = False for root, _, files in os.walk(package.path): for f in fnmatch.filter(files, "*.py"): full_path = os.path.join(root, f) python_files.append(os.path.abspath(full_path)) if file_cmd_exists: for f in files: full_path = os.path.join(root, f) output = subprocess.check_output( ["file", full_path], universal_newlines=True ) # type: str # pylint: disable=unsupported-membership-test if ( "python script" in output or "Python script" in output ) and not f.endswith(".cfg"): # pylint: enable=unsupported-membership-test python_files.append(os.path.abspath(full_path)) python_files = list(OrderedDict.fromkeys(python_files)) print(" {} python files found.".format(len(python_files))) if exceptions: original_file_count = len(python_files) python_files = exceptions.filter_file_exceptions_early( package, python_files ) if original_file_count > len(python_files): print( " After filtering, {} python files will be scanned.".format( len(python_files) ) ) package["python_src"] = python_files
def test_c_discovery_plugin_scan_exceptions(): """Test that the C discovery plugin finds valid C source/header files.""" cdp = CDiscoveryPlugin() package = Package('valid_package', os.path.join(os.path.dirname(__file__), 'valid_package')) exceptions = Exceptions( os.path.join(os.path.dirname(__file__), 'exceptions.yaml')) cdp.scan(package, 'level', exceptions) expected = [ 'test.c', 'test.cpp', 'test.cc', 'test.cxx', 'test.h', 'test.hxx', 'test.hpp' ] if cdp.file_command_exists(): expected += ['oddextensioncpp.source', 'oddextensionc.source'] # We have to add the path to each of the above expected_fullpath = [ os.path.join(package.path, filename) for filename in expected ] # Neat trick to verify that two unordered lists are the same assert set(package['c_src']) == set(expected_fullpath)
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for C files.""" c_files = [] # type: List[str] c_extensions = (".c", ".cc", ".cpp", ".cxx", ".h", ".hxx", ".hpp") file_cmd_exists = True # type: bool if not DiscoveryPlugin.file_command_exists(): file_cmd_exists = False for root, _, files in os.walk(package.path): for f in files: if f.lower().endswith(c_extensions): full_path = os.path.join(root, f) c_files.append(os.path.abspath(full_path)) elif file_cmd_exists: full_path = os.path.join(root, f) output = subprocess.check_output( ["file", full_path], universal_newlines=True) # type: str if ("c source" in output.lower() or "c program" in output.lower() or "c++ source" in output.lower()) and not f.endswith(".cfg"): c_files.append(os.path.abspath(full_path)) c_files = list(OrderedDict.fromkeys(c_files)) print(" {} C/C++ files found.".format(len(c_files))) if exceptions: original_file_count = len(c_files) c_files = exceptions.filter_file_exceptions_early(package, c_files) if original_file_count > len(c_files): print(" After filtering, {} C/C++ files will be scanned.". format(len(c_files))) package["c_src"] = c_files
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for Perl files.""" perl_files = [] # type: List[str] file_cmd_exists = True # type: bool if not DiscoveryPlugin.file_command_exists(): file_cmd_exists = False for root, _, files in os.walk(package.path): for f in fnmatch.filter(files, "*.pl"): full_path = os.path.join(root, f) perl_files.append(os.path.abspath(full_path)) if file_cmd_exists: for f in files: full_path = os.path.join(root, f) output = subprocess.check_output( ["file", full_path], universal_newlines=True ) # type: str if "perl script" in output.lower(): perl_files.append(os.path.abspath(full_path)) perl_files = list(OrderedDict.fromkeys(perl_files)) print(" {} Perl files found.".format(len(perl_files))) if exceptions: original_file_count = len(perl_files) perl_files = exceptions.filter_file_exceptions_early(package, perl_files) if original_file_count > len(perl_files): print( " After filtering, {} perl files will be scanned.".format( len(perl_files) ) ) package["perl_src"] = perl_files