def test_filter_files_with_file_extensions(self):
basedir = "/tmp/lint/"
all_files = [
f"{basedir}src/foo.ext",
f"{basedir}README.md",
f"{basedir}LICENSE",
f"{basedir}target/foo.ext",
]
for (file_extensions, expected) in [
([], []),
([".md"], [f"{basedir}README.md"]),
([""], [f"{basedir}LICENSE"]),
(["", ".md"], [f"{basedir}LICENSE", f"{basedir}README.md"]),
]:
filtered_files = utils.filter_files(
all_files=all_files,
filter_regex_include=None,
filter_regex_exclude=None,
file_names_regex=[],
file_extensions=file_extensions,
ignored_files=[],
ignore_generated_files=False,
)
self.assertListEqual(sorted(filtered_files), sorted(expected),
f"check {file_extensions}")
def collect_files(self):
# Collect not filtered list of files
if self.validate_all_code_base is False:
try:
all_files = self.list_files_git_diff()
except git.InvalidGitRepositoryError as git_err:
logging.warning(
"Unable to list updated files from git diff. Switch to VALIDATE_ALL_CODE_BASE=true"
)
logging.debug(f"git error: {str(git_err)}")
all_files = self.list_files_all()
self.validate_all_code_base = True
else:
all_files = self.list_files_all()
all_files = sorted(set(all_files))
logging.debug("All found files before filtering:\n- %s", "\n- ".join(all_files))
# Filter files according to fileExtensions, fileNames , filterRegexInclude and filterRegexExclude
if len(self.file_extensions) > 0:
logging.info(
"- File extensions: " + ", ".join(sorted(self.file_extensions))
)
if len(self.file_names_regex) > 0:
logging.info(
"- File names (regex): " + ", ".join(sorted(self.file_names_regex))
)
if self.filter_regex_include is not None:
logging.info("- Including regex: " + self.filter_regex_include)
if self.filter_regex_exclude is not None:
logging.info("- Excluding regex: " + self.filter_regex_exclude)
filtered_files = utils.filter_files(
all_files=all_files,
filter_regex_include=self.filter_regex_include,
filter_regex_exclude=self.filter_regex_exclude,
file_names_regex=self.file_names_regex,
file_extensions=self.file_extensions,
)
logging.info(
"Kept ["
+ str(len(filtered_files))
+ "] files on ["
+ str(len(all_files))
+ "] found files"
)
logging.debug(
"Kept files before applying linter filters:\n- %s",
"\n- ".join(filtered_files),
)
# Collect matching files for each linter
for linter in self.linters:
linter.collect_files(filtered_files)
if len(linter.files) == 0 and linter.lint_all_files is False:
linter.is_active = False
def collect_files(self, all_files):
self.log_file_filters()
# Filter all files to keep only the ones matching with the current linter
self.files = utils.filter_files(
all_files=all_files,
filter_regex_include=self.filter_regex_include,
filter_regex_exclude=self.filter_regex_exclude,
file_names_regex=self.file_names_regex,
file_extensions=self.file_extensions,
file_names_not_ends_with=self.file_names_not_ends_with,
file_contains_regex=self.file_contains_regex,
files_sub_directory=self.files_sub_directory,
lint_all_other_linters_files=self.lint_all_other_linters_files,
)
logging.debug(
"%s linter files after applying linter filters:\n- %s",
self.name,
"\n- ".join(self.files),
)
def collect_files(self):
# Collect not filtered list of files
if self.validate_all_code_base is False:
try:
all_files = self.list_files_git_diff()
except git.InvalidGitRepositoryError as git_err:
logging.warning(
"Unable to list updated files from git diff. Switch to VALIDATE_ALL_CODE_BASE=true"
)
logging.debug(f"git error: {str(git_err)}")
all_files = self.list_files_all()
self.validate_all_code_base = True
else:
all_files = self.list_files_all()
all_files = sorted(set(all_files))
logging.debug("All found files before filtering:" +
utils.format_bullet_list(all_files))
# Filter files according to fileExtensions, fileNames , filterRegexInclude and filterRegexExclude
if len(self.file_extensions) > 0:
logging.info("- File extensions: " +
", ".join(sorted(self.file_extensions)))
if len(self.file_names_regex) > 0:
logging.info("- File names (regex): " +
", ".join(sorted(self.file_names_regex)))
if self.filter_regex_include is not None:
logging.info("- Including regex: " + self.filter_regex_include)
if self.filter_regex_exclude is not None:
logging.info("- Excluding regex: " + self.filter_regex_exclude)
# List git ignored files if necessary
ignored_files = []
if self.ignore_gitignore_files is True:
try:
ignored_files = self.list_git_ignored_files()
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug("- Excluding .gitignored files [" +
str(len(ignored_files)) + "]: " +
", ".join(ignored_files))
else:
logging.info("- Excluding .gitignored files [" +
str(len(ignored_files)) + "]: " +
", ".join(ignored_files[0:10]) +
(",...(full list in DEBUG)"
if len(ignored_files) > 10 else ""))
except git.InvalidGitRepositoryError as git_err:
logging.warning(
f"Unable to list git ignored files ({str(git_err)})")
ignored_files = []
except Exception as git_err:
logging.warning(
f"Unable to list git ignored files ({str(git_err)})")
ignored_files = []
# Apply all filters on file list
filtered_files = utils.filter_files(
all_files=all_files,
filter_regex_include=self.filter_regex_include,
filter_regex_exclude=self.filter_regex_exclude,
file_names_regex=self.file_names_regex,
file_extensions=self.file_extensions,
ignored_files=ignored_files,
ignore_generated_files=self.ignore_generated_files,
)
logging.info("Kept [" + str(len(filtered_files)) + "] files on [" +
str(len(all_files)) + "] found files")
logging.debug(
"Kept files before applying linter filters:\n- %s",
"\n- ".join(filtered_files),
)
# Collect matching files for each linter
for linter in self.linters:
linter.collect_files(filtered_files)
if len(linter.files) == 0 and linter.lint_all_files is False:
linter.is_active = False