def lint(paths, config, fix=None, **lintargs):
log = lintargs["log"]
paths = list(expand_exclusions(paths, config, lintargs["root"]))
# We ignored some specific files for a bunch of reasons.
# Not using excluding to avoid duplication
if lintargs.get("use_filters", True):
paths = remove_ignored_path(paths, lintargs["root"], log)
# An empty path array can occur when the user passes in `-n`. If we don't
# return early in this case, rustfmt will attempt to read stdin and hang.
if not paths:
return []
binary = get_clang_format_binary()
if not binary:
print(CLANG_FORMAT_NOT_FOUND)
if "MOZ_AUTOMATION" in os.environ:
return 1
return []
cmd_args = [binary]
base_command = cmd_args + ["--version"]
version = run_process(config, base_command)
log.debug("Version: {}".format(version))
cmd_args.append("--dry-run")
base_command = cmd_args + paths
log.debug("Command: {}".format(" ".join(cmd_args)))
output = run_process(config, base_command)
output_list = []
if len(output) % 3 != 0:
raise Exception(
"clang-format output should be a multiple of 3. Output: %s" % output
)
fixed = (int)(len(output) / 3)
if fix:
cmd_args.remove("--dry-run")
cmd_args.append("-i")
base_command = cmd_args + paths
log.debug("Command: {}".format(" ".join(cmd_args)))
output = run_process(config, base_command)
else:
fixed = 0
for i in range(0, len(output), 3):
# Merge the element 3 by 3 (clang-format output)
line = output[i]
line += ";" + output[i + 1]
line += ";" + output[i + 2]
output_list.append(line)
if fix:
# clang-format is able to fix all issues so don't bother parsing the output.
return {"results": [], "fixed": fixed}
return {"results": parse_issues(config, output_list, paths, log), "fixed": fixed}
def lint(files, config, **lintargs):
log = lintargs["log"]
config["root"] = lintargs["root"]
paths = expand_exclusions(files, config, config["root"])
paths = list(paths)
chunk_size = 50
binary = get_rstcheck_binary()
rstcheck_options = "--ignore-language=cpp,json"
while paths:
cmdargs = [which("python"), binary, rstcheck_options
] + paths[:chunk_size]
log.debug("Command: {}".format(" ".join(cmdargs)))
proc = subprocess.Popen(
cmdargs,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ,
universal_newlines=True,
)
all_errors = proc.communicate()[1]
for errors in all_errors.split("\n"):
if len(errors) > 1:
filename, lineno, level, message = parse_with_split(errors)
res = {
"path": filename,
"message": message,
"lineno": lineno,
"level": "error" if int(level) >= 2 else "warning",
}
results.append(result.from_config(config, **res))
paths = paths[chunk_size:]
return results
def run_linter(python, paths, config, **lintargs):
binary = find_executable(python)
if not binary:
# If we're in automation, this is fatal. Otherwise, the warning in the
# setup method was already printed.
if 'MOZ_AUTOMATION' in os.environ:
return 1
return []
files = expand_exclusions(paths, config, lintargs['root'])
with mozfile.NamedTemporaryFile(mode='w') as fh:
fh.write('\n'.join(files))
fh.flush()
cmd = [binary, os.path.join(here, 'check_compat.py'), fh.name]
proc = PyCompatProcess(config, cmd)
proc.run()
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
return results
def lint(paths, config, fix=None, **lintargs):
results = []
if platform.system() == "Windows":
# Windows doesn't have permissions in files
# Exit now
return results
files = list(expand_exclusions(paths, config, lintargs["root"]))
for f in files:
if os.access(f, os.X_OK):
if config.get("allow-shebang"):
with open(f, "r+") as content:
# Some source files have +x permissions
line = content.readline()
if line.startswith("#!"):
# Check if the file doesn't start with a shebang
# if it does, not a warning
continue
if fix:
# We want to fix it, do it and leave
os.chmod(f, 0o644)
continue
res = {
"path": f,
"message": "Execution permissions on a source file",
"level": "error",
}
results.append(result.from_config(config, **res))
return results
def lint(paths, config, fix=None, **lintargs):
log = lintargs['log']
paths = list(expand_exclusions(paths, config, lintargs['root']))
binary = get_rustfmt_binary()
if is_old_rustfmt(binary):
print(RUSTFMT_DEPRECATED_VERSION)
return 1
if not binary:
print(RUSTFMT_NOT_FOUND)
if "MOZ_AUTOMATION" in os.environ:
return 1
return []
cmd_args = [binary]
if not fix:
cmd_args.append("--check")
base_command = cmd_args + paths
log.debug("Command: {}".format(' '.join(cmd_args)))
output = run_process(config, base_command)
if fix:
# Rustfmt is able to fix all issues so don't bother parsing the output.
return []
return parse_issues(config, output, paths)
def lint(paths, config, fix=None, **lintargs):
if platform.system() == 'Windows':
# Windows doesn't have permissions in files
# Exit now
return results
files = list(expand_exclusions(paths, config, lintargs['root']))
for f in files:
if os.access(f, os.X_OK):
with open(f, 'r+') as content:
# Some source files have +x permissions
line = content.readline()
if line.startswith("#!"):
# Check if the file doesn't start with a shebang
# if it does, not a warning
continue
if fix:
# We want to fix it, do it and leave
os.chmod(f, 0o644)
continue
res = {
'path': f,
'message': "Execution permissions on a source file",
'level': 'error'
}
results.append(result.from_config(config, **res))
return results
def lint(paths, config, **lintargs):
log = lintargs["log"]
binary = get_pylint_binary()
log = lintargs["log"]
paths = list(expand_exclusions(paths, config, lintargs["root"]))
cmd_args = [binary]
results = []
# list from https://code.visualstudio.com/docs/python/linting#_pylint
# And ignore a bit more elements
cmd_args += [
"-fjson",
"--disable=all",
"--enable=F,E,unreachable,duplicate-key,unnecessary-semicolon,global-variable-not-assigned,unused-variable,binary-op-exception,bad-format-string,anomalous-backslash-in-string,bad-open-mode,no-else-return", # NOQA: E501
"--disable=import-error,no-member",
]
base_command = cmd_args + paths
log.debug("Command: {}".format(" ".join(cmd_args)))
log.debug("pylint version: {}".format(get_pylint_version(binary)))
output = " ".join(run_process(config, base_command))
results = parse_issues(log, config, str(output), [])
return results
def lint(paths, config, fix=None, **lintargs):
log = lintargs['log']
paths = list(expand_exclusions(paths, config, lintargs['root']))
# An empty path array can occur when the user passes in `-n`. If we don't
# return early in this case, rustfmt will attempt to read stdin and hang.
if not paths:
return []
binary = get_rustfmt_binary()
if is_old_rustfmt(binary):
print(RUSTFMT_DEPRECATED_VERSION)
return 1
if not binary:
print(RUSTFMT_NOT_FOUND)
if "MOZ_AUTOMATION" in os.environ:
return 1
return []
cmd_args = [binary]
if not fix:
cmd_args.append("--check")
base_command = cmd_args + paths
log.debug("Command: {}".format(' '.join(cmd_args)))
output = run_process(config, base_command)
if fix:
# Rustfmt is able to fix all issues so don't bother parsing the output.
return []
return parse_issues(config, output, paths)
def run_linter(python, paths, config, **lintargs):
log = lintargs["log"]
binary = find_executable(python)
if not binary:
# If we're in automation, this is fatal. Otherwise, the warning in the
# setup method was already printed.
if "MOZ_AUTOMATION" in os.environ:
return 1
return []
files = expand_exclusions(paths, config, lintargs["root"])
with mozfile.NamedTemporaryFile(mode="w") as fh:
fh.write("\n".join(files))
fh.flush()
cmd = [binary, os.path.join(here, "check_compat.py"), fh.name]
log.debug("Command: {}".format(" ".join(cmd)))
proc = PyCompatProcess(config, cmd)
proc.run()
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
return results
def lint(files, config, **lintargs):
config['root'] = lintargs['root']
paths = expand_exclusions(files, config, config['root'])
paths = list(paths)
chunk_size = 50
binary = get_rstcheck_binary()
while paths:
cmdargs = [
binary,
] + paths[:chunk_size]
proc = subprocess.Popen(cmdargs,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ)
all_errors = proc.communicate()[1]
for errors in all_errors.split("\n"):
if len(errors) > 1:
filename, lineno, level, message = parse_with_split(errors)
res = {
'path': filename,
'message': message,
'lineno': lineno,
'level': "error" if int(level) >= 2 else "warning",
}
results.append(result.from_config(config, **res))
paths = paths[chunk_size:]
return results
def wrap_make_file_checker_manager(self):
"""Flake8 is very inefficient when it comes to applying exclusion
rules, using `expand_exclusions` to turn directories into a list of
relevant python files is an order of magnitude faster.
Hooking into flake8 here also gives us a convenient place to merge the
`exclude` rules specified in the root .flake8 with the ones added by
tools/lint/mach_commands.py.
"""
# Ignore exclude rules if `--no-filter` was passed in.
config.setdefault('exclude', [])
if lintargs.get('use_filters', True):
config['exclude'].extend(self.options.exclude)
# Since we use the root .flake8 file to store exclusions, we haven't
# properly filtered the paths through mozlint's `filterpaths` function
# yet. This mimics that though there could be other edge cases that are
# different. Maybe we should call `filterpaths` directly, though for
# now that doesn't appear to be necessary.
filtered = [
p for p in paths
if not any(p.startswith(e) for e in config['exclude'])
]
self.args = self.args + list(expand_exclusions(filtered, config, root))
if not self.args:
raise NothingToLint
return orig_make_file_checker_manager()
def lint(paths, config, fix=None, **lintargs):
files = list(expand_exclusions(paths, config, lintargs["root"]))
return run_black(
config,
files,
fix=fix,
log=lintargs["log"],
virtualenv_bin_path=lintargs.get("virtualenv_bin_path"),
)
def checkdupes(paths, config, **kwargs):
results = []
errors = []
pref_names = get_names(config["support-files"][0])
files = list(expand_exclusions(paths, config, kwargs["root"]))
for file in files:
errors.extend(check_against(file, pref_names))
for error in errors:
results.append(result.from_config(config, **error))
return results
def lint(paths, config, fix=None, **lintargs):
files = list(expand_exclusions(paths, config, lintargs['root']))
# to retrieve the future changes
results = run_rustfmt(config, files, fix=False)
if fix and results:
# To do the actual change
run_rustfmt(config, files, fix=True)
return results
def lint(paths, config, fix=None, **lintargs):
files = list(expand_exclusions(paths, config, lintargs["root"]))
exclusions = get_exclusions(lintargs["root"])
results = []
for path in files:
contents = open(path, "r", encoding="utf-8").read()
linter = Linter(path, config, exclusions, contents,
get_offsets_and_lines(contents))
linter.visit(parse(contents))
results.extend(linter.results)
return results
def lint(paths, config, fix=None, **lintargs):
files = list(expand_exclusions(paths, config, lintargs["root"]))
for f in files:
with open(f, "rb") as open_file:
hasFix = False
content_to_write = []
for i, line in enumerate(open_file):
if line.endswith(b" \n"):
# We found a trailing whitespace
if fix:
# We want to fix it, strip the trailing spaces
content_to_write.append(line.rstrip() + b"\n")
hasFix = True
else:
res = {
"path": f,
"message": "Trailing whitespace",
"level": "error",
"lineno": i + 1,
}
results.append(result.from_config(config, **res))
else:
if fix:
content_to_write.append(line)
if hasFix:
# Only update the file when we found a change to make
with open(f, "wb") as open_file_to_write:
open_file_to_write.write(b"".join(content_to_write))
# We are still using the same fp, let's return to the first
# line
open_file.seek(0)
# Open it as once as we just need to know if there is
# at least one \r\n
content = open_file.read()
if b"\r\n" in content:
if fix:
# replace \r\n by \n
content = content.replace(b"\r\n", b"\n")
with open(f, "wb") as open_file_to_write:
open_file_to_write.write(content)
else:
res = {
"path": f,
"message": "Windows line return",
"level": "error",
}
results.append(result.from_config(config, **res))
return results
def lint(paths, config, fix=None, **lintargs):
log = lintargs["log"]
paths = list(expand_exclusions(paths, config, lintargs["root"]))
# An empty path array can occur when the user passes in `-n`. If we don't
# return early in this case, rustfmt will attempt to read stdin and hang.
if not paths:
return []
binary = get_rustfmt_binary()
if not binary:
print(RUSTFMT_NOT_FOUND)
if "MOZ_AUTOMATION" in os.environ:
return 1
return []
min_version_str = config.get("min_rustfmt_version")
min_version = StrictVersion(min_version_str)
actual_version = get_rustfmt_version(binary)
log.debug(
"Found version: {}. Minimal expected version: {}".format(
actual_version, min_version
)
)
if actual_version < min_version:
print(RUSTFMT_WRONG_VERSION.format(version=min_version_str))
return 1
cmd_args = [binary]
cmd_args.append("--check")
base_command = cmd_args + paths
log.debug("Command: {}".format(" ".join(cmd_args)))
output = run_process(config, base_command)
issues = parse_issues(config, output, paths)
if fix:
issues["fixed"] = len(issues["results"])
issues["results"] = []
cmd_args.remove("--check")
base_command = cmd_args + paths
log.debug("Command: {}".format(" ".join(cmd_args)))
output = run_process(config, base_command)
return issues
def wrap_make_file_checker_manager(self):
"""Flake8 is very inefficient when it comes to applying exclusion
rules, using `expand_exclusions` to turn directories into a list of
relevant python files is an order of magnitude faster.
Hooking into flake8 here also gives us a convenient place to merge the
`exclude` rules specified in the root .flake8 with the ones added by
tools/lint/mach_commands.py.
"""
config.setdefault('exclude', []).extend(self.options.exclude)
self.options.exclude = None
self.args = self.args + list(expand_exclusions(paths, config, root))
if not self.args:
raise NothingToLint
return orig_make_file_checker_manager()
def lint(paths, config, fix=None, **lintargs):
files = list(expand_exclusions(paths, config, lintargs["root"]))
licenses = load_valid_license()
for f in files:
if is_test(f):
# For now, do not do anything with test (too many)
continue
if not is_valid_license(licenses, f):
res = {
"path": f,
"message": "No matching license strings found in tools/lint/license/valid-licenses.txt", # noqa
"level": "error",
}
results.append(result.from_config(config, **res))
if fix:
fix_me(f)
return results
def lint(paths, config, fix=None, **lintargs):
log = lintargs['log']
paths = list(expand_exclusions(paths, config, lintargs['root']))
# An empty path array can occur when the user passes in `-n`. If we don't
# return early in this case, rustfmt will attempt to read stdin and hang.
if not paths:
return []
binary = get_rustfmt_binary()
min_version_str = config.get('min_rustfmt_version')
min_version = StrictVersion(min_version_str)
actual_version = get_rustfmt_version(binary)
log.debug("Found version: {}. Minimal expected version: {}".format(
actual_version, min_version))
if actual_version < min_version:
print(RUSTFMT_WRONG_VERSION.format(version=min_version_str))
return 1
if not binary:
print(RUSTFMT_NOT_FOUND)
if "MOZ_AUTOMATION" in os.environ:
return 1
return []
cmd_args = [binary]
if not fix:
cmd_args.append("--check")
base_command = cmd_args + paths
log.debug("Command: {}".format(' '.join(cmd_args)))
output = run_process(config, base_command)
if fix:
# Rustfmt is able to fix all issues so don't bother parsing the output.
return []
return parse_issues(config, output, paths)
def test_expand_exclusions(test):
expected = test.pop('expected', [])
paths = list(
pathutils.expand_exclusions(test['paths'], test['config'], root))
assert_paths(paths, expected)
def lint(paths, config, fix=None, **lintargs):
files = list(expand_exclusions(paths, config, lintargs["root"]))
log = lintargs["log"]
for f in files:
with open(f, "rb") as open_file:
hasFix = False
content_to_write = []
try:
lines = open_file.readlines()
# Check for Empty spaces or newline character at end of file
if lines[:].__len__() != 0 and lines[-1:][0].strip().__len__(
) == 0:
# return file pointer to first
open_file.seek(0)
if fix:
# fix Empty lines at end of file
for i, line in reversed(list(enumerate(open_file))):
# determine if line is empty
if line.strip() != b"":
with open(f, "wb") as write_file:
# determine if file's last line have \n, if not then add a \n
if not lines[i].endswith(b"\n"):
lines[i] = lines[i] + b"\n"
# write content to file
for e in lines[:i + 1]:
write_file.write(e)
# end the loop
break
else:
res = {
"path": f,
"message": "Empty Lines at end of file",
"level": "error",
"lineno": open_file.readlines()[:].__len__(),
}
results.append(result.from_config(config, **res))
except Exception as ex:
log.debug("Error: " + str(ex) + ", in file: " + f)
# return file pointer to first
open_file.seek(0)
for i, line in enumerate(open_file):
if line.endswith(b" \n"):
# We found a trailing whitespace
if fix:
# We want to fix it, strip the trailing spaces
content_to_write.append(line.rstrip() + b"\n")
hasFix = True
else:
res = {
"path": f,
"message": "Trailing whitespace",
"level": "error",
"lineno": i + 1,
}
results.append(result.from_config(config, **res))
else:
if fix:
content_to_write.append(line)
if hasFix:
# Only update the file when we found a change to make
with open(f, "wb") as open_file_to_write:
open_file_to_write.write(b"".join(content_to_write))
# We are still using the same fp, let's return to the first
# line
open_file.seek(0)
# Open it as once as we just need to know if there is
# at least one \r\n
content = open_file.read()
if b"\r\n" in content:
if fix:
# replace \r\n by \n
content = content.replace(b"\r\n", b"\n")
with open(f, "wb") as open_file_to_write:
open_file_to_write.write(content)
else:
res = {
"path": f,
"message": "Windows line return",
"level": "error",
}
results.append(result.from_config(config, **res))
return results
def test_expand_exclusions(test):
expected = test.pop("expected", [])
paths = list(
pathutils.expand_exclusions(test["paths"], test["config"], root))
assert_paths(paths, expected)
def lint(paths, config, fix=None, **lintargs):
files = list(expand_exclusions(paths, config, lintargs["root"]))
return run_black(config, files, fix=fix, log=lintargs["log"])