def scan(self,
package: Package,
level: str,
exceptions: Exceptions = None) -> None:
"""Scan package looking for XML files."""
xml_files = [] # type: List[str]
globs = ["*.xml", "*.launch"] # type: List[str]
root = ""
for root, _, files in os.walk(package.path):
for glob in globs:
for f in fnmatch.filter(files, glob):
full_path = os.path.join(root, f)
xml_files.append(os.path.abspath(full_path))
xml_files = list(OrderedDict.fromkeys(xml_files))
print(" {} XML files found.".format(len(xml_files)))
if exceptions:
original_file_count = len(xml_files)
xml_files = exceptions.filter_file_exceptions_early(
package, xml_files)
if original_file_count > len(xml_files):
print(
" After filtering, {} XML files will be scanned.".format(
len(xml_files)))
package["xml"] = xml_files
def scan(self,
package: Package,
level: str,
exceptions: Exceptions = None) -> None:
"""Scan package looking for maven files."""
top_poms = [] # type: List[str]
all_poms = [] # type: List[str]
deepest_pom_level = 999999
for root, _, files in os.walk(package.path):
for f in fnmatch.filter(files, "pom.xml"):
full_path = os.path.join(root, f)
# Kind of an ugly hack, but it makes sure long paths don't
# mess up our depth tracking
if exceptions and not exceptions.filter_file_exceptions_early(
package, [full_path]):
continue
depth = full_path.count(os.sep)
if depth < deepest_pom_level:
deepest_pom_level = depth
top_poms = []
if depth == deepest_pom_level:
top_poms.append(full_path)
all_poms.append(full_path)
top_poms = list(OrderedDict.fromkeys(top_poms))
all_poms = list(OrderedDict.fromkeys(all_poms))
print(" {} Maven POM files found.".format(len(all_poms)))
print(" {} top-level Maven POM files found.".format(len(top_poms)))
package["all_poms"] = all_poms
package["top_poms"] = top_poms
def test_filter_file_exceptions_early_onlyall():
"""
Test that filter_file_exceptions_early only uses exceptions with tools=all.
Expected result: No change to the files list
"""
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), 'early_exceptions.yaml'))
package = Package('test', os.path.dirname(__file__))
files = [os.path.join(os.path.dirname(__file__), 'uncommontext')]
filtered_files = exceptions.filter_file_exceptions_early(package, files)
assert filtered_files == files
def test_filter_file_exceptions_early():
"""
Test that filter_file_exceptions_early excludes files.
Expected result: Empty files list.
"""
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), 'early_exceptions.yaml'))
package = Package('test', os.path.dirname(__file__))
files = [os.path.join(os.path.dirname(__file__), 'unlikelystring')]
filtered_files = exceptions.filter_file_exceptions_early(package, files)
assert not filtered_files
def scan(self,
package: Package,
level: str,
exceptions: Exceptions = None) -> None:
"""Scan package looking for TeX files."""
tex_files = [] # type: List[str]
globs = ["*.tex", "*.bib"] # type: List[str]
file_cmd_exists = True # type: bool
if not DiscoveryPlugin.file_command_exists():
file_cmd_exists = False
root = "" # type: str
for root, _, files in os.walk(package.path):
for glob in globs:
for f in fnmatch.filter(files, glob):
full_path = os.path.join(root, f)
tex_files.append(os.path.abspath(full_path))
if file_cmd_exists:
for f in files:
full_path = os.path.join(root, f)
output = subprocess.check_output(["file", full_path],
universal_newlines=True)
if f.endswith(".sty") or f.endswith(".log") or f.endswith(
".cls"):
continue
# pylint: disable=unsupported-membership-test
if ("LaTeX document" in output
or "BibTeX text file" in output
or "LaTeX 2e document" in output):
# pylint: enable=unsupported-membership-test
tex_files.append(os.path.abspath(full_path))
tex_files = list(OrderedDict.fromkeys(tex_files))
print(" {} TeX files found.".format(len(tex_files)))
if exceptions:
original_file_count = len(tex_files) # type: int
tex_files = exceptions.filter_file_exceptions_early(
package, tex_files)
if original_file_count > len(tex_files):
print(
" After filtering, {} TeX files will be scanned.".format(
len(tex_files)))
package["tex"] = tex_files
def test_filter_file_exceptions_early():
"""
Test that filter_file_exceptions_early excludes files.
Expected result: Empty files list.
"""
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), "early_exceptions.yaml"))
package = Package("test", os.path.dirname(__file__))
files = [
"/home/travis/build/unlikelystring",
os.path.join(os.path.dirname(__file__), "unlikelystring"),
]
filtered_files = exceptions.filter_file_exceptions_early(package, files)
assert not filtered_files
def test_filter_file_exceptions_early_dupes():
"""
Test that filter_file_exceptions_early excludes duplicated files.
I have no idea why one might have duplicate files, but might as well test it!
Expected result: Empty file list.
"""
exceptions = Exceptions(
os.path.join(os.path.dirname(__file__), 'early_exceptions.yaml'))
package = Package('test', os.path.dirname(__file__))
files = [
os.path.join(os.path.dirname(__file__), 'unlikelystring'),
os.path.join(os.path.dirname(__file__), 'unlikelystring')
]
filtered_files = exceptions.filter_file_exceptions_early(package, files)
assert not filtered_files
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None:
"""Scan package looking for python files."""
python_files = [] # type: List[str]
file_cmd_exists = True # type: bool
if not DiscoveryPlugin.file_command_exists():
file_cmd_exists = False
for root, _, files in os.walk(package.path):
for f in fnmatch.filter(files, "*.py"):
full_path = os.path.join(root, f)
python_files.append(os.path.abspath(full_path))
if file_cmd_exists:
for f in files:
full_path = os.path.join(root, f)
output = subprocess.check_output(
["file", full_path], universal_newlines=True
) # type: str
# pylint: disable=unsupported-membership-test
if (
"python script" in output or "Python script" in output
) and not f.endswith(".cfg"):
# pylint: enable=unsupported-membership-test
python_files.append(os.path.abspath(full_path))
python_files = list(OrderedDict.fromkeys(python_files))
print(" {} python files found.".format(len(python_files)))
if exceptions:
original_file_count = len(python_files)
python_files = exceptions.filter_file_exceptions_early(
package, python_files
)
if original_file_count > len(python_files):
print(
" After filtering, {} python files will be scanned.".format(
len(python_files)
)
)
package["python_src"] = python_files
def scan(self,
package: Package,
level: str,
exceptions: Exceptions = None) -> None:
"""Scan package looking for C files."""
c_files = [] # type: List[str]
c_extensions = (".c", ".cc", ".cpp", ".cxx", ".h", ".hxx", ".hpp")
file_cmd_exists = True # type: bool
if not DiscoveryPlugin.file_command_exists():
file_cmd_exists = False
for root, _, files in os.walk(package.path):
for f in files:
if f.lower().endswith(c_extensions):
full_path = os.path.join(root, f)
c_files.append(os.path.abspath(full_path))
elif file_cmd_exists:
full_path = os.path.join(root, f)
output = subprocess.check_output(
["file", full_path],
universal_newlines=True) # type: str
if ("c source" in output.lower()
or "c program" in output.lower() or "c++ source"
in output.lower()) and not f.endswith(".cfg"):
c_files.append(os.path.abspath(full_path))
c_files = list(OrderedDict.fromkeys(c_files))
print(" {} C/C++ files found.".format(len(c_files)))
if exceptions:
original_file_count = len(c_files)
c_files = exceptions.filter_file_exceptions_early(package, c_files)
if original_file_count > len(c_files):
print(" After filtering, {} C/C++ files will be scanned.".
format(len(c_files)))
package["c_src"] = c_files
def scan(self, package: Package, level: str, exceptions: Exceptions = None) -> None:
"""Scan package looking for Perl files."""
perl_files = [] # type: List[str]
file_cmd_exists = True # type: bool
if not DiscoveryPlugin.file_command_exists():
file_cmd_exists = False
for root, _, files in os.walk(package.path):
for f in fnmatch.filter(files, "*.pl"):
full_path = os.path.join(root, f)
perl_files.append(os.path.abspath(full_path))
if file_cmd_exists:
for f in files:
full_path = os.path.join(root, f)
output = subprocess.check_output(
["file", full_path], universal_newlines=True
) # type: str
if "perl script" in output.lower():
perl_files.append(os.path.abspath(full_path))
perl_files = list(OrderedDict.fromkeys(perl_files))
print(" {} Perl files found.".format(len(perl_files)))
if exceptions:
original_file_count = len(perl_files)
perl_files = exceptions.filter_file_exceptions_early(package, perl_files)
if original_file_count > len(perl_files):
print(
" After filtering, {} perl files will be scanned.".format(
len(perl_files)
)
)
package["perl_src"] = perl_files