def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for XML files.""" xml_files = [] # type: List[str] globs = ["*.xml", "*.launch"] # type: List[str] root = "" for root, _, files in os.walk(package.path): for glob in globs: for f in fnmatch.filter(files, glob): full_path = os.path.join(root, f) xml_files.append(os.path.abspath(full_path)) xml_files = list(OrderedDict.fromkeys(xml_files)) print(" {} XML files found.".format(len(xml_files))) if exceptions: original_file_count = len(xml_files) xml_files = exceptions.filter_file_exceptions_early( package, xml_files) if original_file_count > len(xml_files): print( " After filtering, {} XML files will be scanned.".format( len(xml_files))) package["xml"] = xml_files
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for maven files.""" top_poms = [] # type: List[str] all_poms = [] # type: List[str] deepest_pom_level = 999999 for root, _, files in os.walk(package.path): for f in fnmatch.filter(files, "pom.xml"): full_path = os.path.join(root, f) # Kind of an ugly hack, but it makes sure long paths don't # mess up our depth tracking if exceptions and not exceptions.filter_file_exceptions_early( package, [full_path]): continue depth = full_path.count(os.sep) if depth < deepest_pom_level: deepest_pom_level = depth top_poms = [] if depth == deepest_pom_level: top_poms.append(full_path) all_poms.append(full_path) top_poms = list(OrderedDict.fromkeys(top_poms)) all_poms = list(OrderedDict.fromkeys(all_poms)) print(" {} Maven POM files found.".format(len(all_poms))) print(" {} top-level Maven POM files found.".format(len(top_poms))) package["all_poms"] = all_poms package["top_poms"] = top_poms
def test_filter_file_exceptions_early_onlyall(): """ Test that filter_file_exceptions_early only uses exceptions with tools=all. Expected result: No change to the files list """ exceptions = Exceptions( os.path.join(os.path.dirname(__file__), 'early_exceptions.yaml')) package = Package('test', os.path.dirname(__file__)) files = [os.path.join(os.path.dirname(__file__), 'uncommontext')] filtered_files = exceptions.filter_file_exceptions_early(package, files) assert filtered_files == files
def test_filter_file_exceptions_early(): """ Test that filter_file_exceptions_early excludes files. Expected result: Empty files list. """ exceptions = Exceptions( os.path.join(os.path.dirname(__file__), 'early_exceptions.yaml')) package = Package('test', os.path.dirname(__file__)) files = [os.path.join(os.path.dirname(__file__), 'unlikelystring')] filtered_files = exceptions.filter_file_exceptions_early(package, files) assert not filtered_files
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for TeX files.""" tex_files = [] # type: List[str] globs = ["*.tex", "*.bib"] # type: List[str] file_cmd_exists = True # type: bool if not DiscoveryPlugin.file_command_exists(): file_cmd_exists = False root = "" # type: str for root, _, files in os.walk(package.path): for glob in globs: for f in fnmatch.filter(files, glob): full_path = os.path.join(root, f) tex_files.append(os.path.abspath(full_path)) if file_cmd_exists: for f in files: full_path = os.path.join(root, f) output = subprocess.check_output(["file", full_path], universal_newlines=True) if f.endswith(".sty") or f.endswith(".log") or f.endswith( ".cls"): continue # pylint: disable=unsupported-membership-test if ("LaTeX document" in output or "BibTeX text file" in output or "LaTeX 2e document" in output): # pylint: enable=unsupported-membership-test tex_files.append(os.path.abspath(full_path)) tex_files = list(OrderedDict.fromkeys(tex_files)) print(" {} TeX files found.".format(len(tex_files))) if exceptions: original_file_count = len(tex_files) # type: int tex_files = exceptions.filter_file_exceptions_early( package, tex_files) if original_file_count > len(tex_files): print( " After filtering, {} TeX files will be scanned.".format( len(tex_files))) package["tex"] = tex_files
def test_filter_file_exceptions_early(): """ Test that filter_file_exceptions_early excludes files. Expected result: Empty files list. """ exceptions = Exceptions( os.path.join(os.path.dirname(__file__), "early_exceptions.yaml")) package = Package("test", os.path.dirname(__file__)) files = [ "/home/travis/build/unlikelystring", os.path.join(os.path.dirname(__file__), "unlikelystring"), ] filtered_files = exceptions.filter_file_exceptions_early(package, files) assert not filtered_files
def test_filter_file_exceptions_early_dupes(): """ Test that filter_file_exceptions_early excludes duplicated files. I have no idea why one might have duplicate files, but might as well test it! Expected result: Empty file list. """ exceptions = Exceptions( os.path.join(os.path.dirname(__file__), 'early_exceptions.yaml')) package = Package('test', os.path.dirname(__file__)) files = [ os.path.join(os.path.dirname(__file__), 'unlikelystring'), os.path.join(os.path.dirname(__file__), 'unlikelystring') ] filtered_files = exceptions.filter_file_exceptions_early(package, files) assert not filtered_files
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for python files.""" python_files = [] # type: List[str] file_cmd_exists = True # type: bool if not DiscoveryPlugin.file_command_exists(): file_cmd_exists = False for root, _, files in os.walk(package.path): for f in fnmatch.filter(files, "*.py"): full_path = os.path.join(root, f) python_files.append(os.path.abspath(full_path)) if file_cmd_exists: for f in files: full_path = os.path.join(root, f) output = subprocess.check_output( ["file", full_path], universal_newlines=True ) # type: str # pylint: disable=unsupported-membership-test if ( "python script" in output or "Python script" in output ) and not f.endswith(".cfg"): # pylint: enable=unsupported-membership-test python_files.append(os.path.abspath(full_path)) python_files = list(OrderedDict.fromkeys(python_files)) print(" {} python files found.".format(len(python_files))) if exceptions: original_file_count = len(python_files) python_files = exceptions.filter_file_exceptions_early( package, python_files ) if original_file_count > len(python_files): print( " After filtering, {} python files will be scanned.".format( len(python_files) ) ) package["python_src"] = python_files
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for C files.""" c_files = [] # type: List[str] c_extensions = (".c", ".cc", ".cpp", ".cxx", ".h", ".hxx", ".hpp") file_cmd_exists = True # type: bool if not DiscoveryPlugin.file_command_exists(): file_cmd_exists = False for root, _, files in os.walk(package.path): for f in files: if f.lower().endswith(c_extensions): full_path = os.path.join(root, f) c_files.append(os.path.abspath(full_path)) elif file_cmd_exists: full_path = os.path.join(root, f) output = subprocess.check_output( ["file", full_path], universal_newlines=True) # type: str if ("c source" in output.lower() or "c program" in output.lower() or "c++ source" in output.lower()) and not f.endswith(".cfg"): c_files.append(os.path.abspath(full_path)) c_files = list(OrderedDict.fromkeys(c_files)) print(" {} C/C++ files found.".format(len(c_files))) if exceptions: original_file_count = len(c_files) c_files = exceptions.filter_file_exceptions_early(package, c_files) if original_file_count > len(c_files): print(" After filtering, {} C/C++ files will be scanned.". format(len(c_files))) package["c_src"] = c_files
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None: """Scan package looking for Perl files.""" perl_files = [] # type: List[str] file_cmd_exists = True # type: bool if not DiscoveryPlugin.file_command_exists(): file_cmd_exists = False for root, _, files in os.walk(package.path): for f in fnmatch.filter(files, "*.pl"): full_path = os.path.join(root, f) perl_files.append(os.path.abspath(full_path)) if file_cmd_exists: for f in files: full_path = os.path.join(root, f) output = subprocess.check_output( ["file", full_path], universal_newlines=True ) # type: str if "perl script" in output.lower(): perl_files.append(os.path.abspath(full_path)) perl_files = list(OrderedDict.fromkeys(perl_files)) print(" {} Perl files found.".format(len(perl_files))) if exceptions: original_file_count = len(perl_files) perl_files = exceptions.filter_file_exceptions_early(package, perl_files) if original_file_count > len(perl_files): print( " After filtering, {} perl files will be scanned.".format( len(perl_files) ) ) package["perl_src"] = perl_files