def _check_all(cmd, all_filenames):
# Output from clang-tidy can be massive, so only run 16 instances
# of it at a time. Process and drop that output before running more
for filenames in lintutils.chunk(all_filenames, 16**2):
# each clang-tidy instance will process 16 files
chunks = list(lintutils.chunk(filenames, 16))
cmds = [cmd + some for some in chunks]
results = lintutils.run_parallel(cmds, stderr=DEVNULL, stdout=PIPE)
error = False
# record completed processes (keyed by the first filename in the input chunk)
# for lookup in _check_some_files
completed_processes = {
_get_chunk_key(some): result
for some, result in zip(chunks, results)
}
checker = partial(_check_some_files, completed_processes)
pool = mp.Pool()
try:
# check output of completed clang-tidy invocations in parallel
for problem_files, stdout in pool.imap(checker, chunks):
if problem_files:
msg = "clang-tidy suggested fixes for {}"
print("\n".join(map(msg.format, problem_files)))
error = True
except Exception:
error = True
raise
finally:
pool.terminate()
pool.join()
if error:
sys.exit(1)
def _check_all(cmd, filenames):
# each clang-tidy instance will process 16 files
chunks = lintutils.chunk(filenames, 16)
cmds = [cmd + some for some in chunks]
results = lintutils.run_parallel(cmds, stderr=PIPE, stdout=PIPE)
error = False
# record completed processes (keyed by the first filename in the input
# chunk) for lookup in _check_some_files
completed_processes = {
_get_chunk_key(some): result
for some, result in zip(chunks, results)
}
checker = partial(_check_some_files, completed_processes)
pool = mp.Pool()
try:
# check output of completed clang-tidy invocations in parallel
for problem_files, stdout in pool.imap(checker, chunks):
if problem_files:
msg = "clang-tidy suggested fixes for {}"
print("\n".join(map(msg.format, problem_files)))
error = True
except Exception:
error = True
raise
finally:
pool.terminate()
pool.join()
if error:
sys.exit(1)
def _check_all(cmd, filenames):
# each clang-tidy instance will process 16 files
chunks = lintutils.chunk(filenames, 16)
cmds = [cmd + some for some in chunks]
results = lintutils.run_parallel(cmds, stderr=PIPE, stdout=PIPE)
error = False
# record completed processes (keyed by the first filename in the input
# chunk) for lookup in _check_some_files
completed_processes = {
_get_chunk_key(some): result
for some, result in zip(chunks, results)
}
checker = partial(_check_some_files, completed_processes)
pool = mp.Pool()
try:
# check output of completed clang-tidy invocations in parallel
for problem_files, stdout in pool.imap(checker, chunks):
if problem_files:
msg = "clang-tidy suggested fixes for {}"
print("\n".join(map(msg.format, problem_files)))
error = True
except Exception:
error = True
raise
finally:
pool.terminate()
pool.join()
if error:
sys.exit(1)
def _check_dir(arguments, source_dir, exclude_globs):
formatted_filenames = []
for path in lintutils.get_sources(source_dir, exclude_globs):
formatted_filenames.append(str(path))
if arguments.fix:
if not arguments.quiet:
print("\n".join(
map(lambda x: "Formatting {}".format(x), formatted_filenames)))
# Break clang-format invocations into chunks: each invocation formats
# 16 files. Wait for all processes to complete
results = lintutils.run_parallel(
[[arguments.clang_format_binary, "-style=file", "-i"] + some
for some in lintutils.chunk(formatted_filenames, 16)])
for returncode, stdout, stderr in results:
# if any clang-format reported a parse error, bubble it
if returncode != 0:
sys.exit(returncode)
else:
# run an instance of clang-format for each source file in parallel,
# then wait for all processes to complete
results = lintutils.run_parallel(
[[arguments.clang_format_binary, "-style=file", filename]
for filename in formatted_filenames],
stdout=PIPE,
stderr=PIPE)
checker_args = []
for filename, res in zip(formatted_filenames, results):
# if any clang-format reported a parse error, bubble it
returncode, stdout, stderr = res
if returncode != 0:
print(stderr)
sys.exit(returncode)
checker_args.append((filename, stdout))
error = False
pool = mp.Pool()
try:
# check the output from each invocation of clang-format in parallel
for filename, diff in pool.starmap(_check_one_file, checker_args):
if not arguments.quiet:
print("Checking {}".format(filename))
if diff:
print("{} had clang-format style issues".format(filename))
# Print out the diff to stderr
error = True
# pad with a newline
print(file=sys.stderr)
sys.stderr.writelines(diff)
except Exception:
error = True
raise
finally:
pool.terminate()
pool.join()
sys.exit(1 if error else 0)
def _check_all(cmd, filenames, ignore_checks):
# each clang-tidy instance will process 16 files
chunks = lintutils.chunk(filenames, 16)
cmds = [cmd + some for some in chunks]
results = lintutils.run_parallel(cmds, stderr=PIPE, stdout=PIPE)
error = False
# record completed processes (keyed by the first filename in the input
# chunk) for lookup in _check_some_files
completed_processes = {
_get_chunk_key(some): result
for some, result in zip(chunks, results)
}
checker = partial(_check_some_files, completed_processes)
pool = mp.Pool()
error = False
try:
cnt_error = 0
cnt_warning = 0
cnt_ignore = 0
# check output of completed clang-tidy invocations in parallel
for problem_files, stdout in pool.imap(checker, chunks):
if problem_files:
msg = "clang-tidy suggested fixes for {}"
print("\n".join(map(msg.format, problem_files)))
# ignore thirdparty header file not found issue, such as:
# error: 'fiu.h' file not found [clang-diagnostic-error]
cnt_info = ""
for line in stdout.splitlines():
if any([len(re.findall(check, line)) > 0 for check in ignore_checks]):
cnt_info += line.replace(" error: ", " ignore: ").decode("utf-8") + "\n"
else:
cnt_info += line.decode("utf-8") + "\n"
cnt_error += _count_key(cnt_info, " error: ")
cnt_warning += _count_key(cnt_info, " warning: ")
cnt_ignore += _count_key(cnt_info, " ignore: ")
print(cnt_info)
print("clang-tidy - error: {}, warning: {}, ignore {}".
format(cnt_error, cnt_warning, cnt_ignore))
error = error or (cnt_error > 0 or cnt_warning > 0)
except Exception:
error = True
raise
finally:
pool.terminate()
pool.join()
if error:
sys.exit(1)
exclude_globs.append(line.strip())
formatted_filenames = []
for path in lintutils.get_sources(arguments.source_dir, exclude_globs):
formatted_filenames.append(str(path))
if arguments.fix:
if not arguments.quiet:
print("\n".join(
map(lambda x: "Formatting {}".format(x), formatted_filenames)))
# Break clang-format invocations into chunks: each invocation formats
# 16 files. Wait for all processes to complete
results = lintutils.run_parallel(
[[arguments.clang_format_binary, "-i"] + some
for some in lintutils.chunk(formatted_filenames, 16)])
for returncode, stdout, stderr in results:
# if any clang-format reported a parse error, bubble it
if returncode != 0:
sys.exit(returncode)
else:
# run an instance of clang-format for each source file in parallel,
# then wait for all processes to complete
results = lintutils.run_parallel(
[[arguments.clang_format_binary, filename]
for filename in formatted_filenames],
stdout=PIPE,
stderr=PIPE)
for returncode, stdout, stderr in results:
# if any clang-format reported a parse error, bubble it
'--linelength=90',
'--filter=' + ','.join(_filters)
]
if (arguments.cpplint_binary.endswith('.py') and
platform.system() == 'Windows'):
# Windows doesn't support executable scripts; execute with
# sys.executable
cmd.insert(0, sys.executable)
if arguments.quiet:
cmd.append('--quiet')
else:
print("\n".join(map(lambda x: "Linting {}".format(x),
linted_filenames)))
# lint files in chunks: each invocation of cpplint will process 16 files
chunks = lintutils.chunk(linted_filenames, 16)
cmds = [cmd + some for some in chunks]
results = lintutils.run_parallel(cmds, stdout=PIPE, stderr=STDOUT)
error = False
# record completed processes (keyed by the first filename in the input
# chunk) for lookup in _check_some_files
completed_processes = {
_get_chunk_key(filenames): result
for filenames, result in zip(chunks, results)
}
checker = partial(_check_some_files, completed_processes)
pool = mp.Pool()
try:
# scan the outputs of various cpplint invocations in parallel to
# distill a list of problematic files
parser.add_argument("--quiet",
default=False,
action="store_true",
help="If specified, only print errors")
arguments = parser.parse_args()
exclude_globs = []
if arguments.exclude_globs:
for line in open(arguments.exclude_globs):
exclude_globs.append(line.strip())
linted_filenames = []
for path in lintutils.get_sources(arguments.source_dir, exclude_globs):
linted_filenames.append(path)
if not arguments.quiet:
msg = 'Tidying {}' if arguments.fix else 'Checking {}'
print("\n".join(map(msg.format, linted_filenames)))
cmd = [arguments.clang_tidy_binary, '-p', arguments.compile_commands]
if arguments.fix:
cmd.append('-fix')
results = lintutils.run_parallel(
[cmd + some for some in lintutils.chunk(linted_filenames, 16)])
for returncode, stdout, stderr in results:
if returncode != 0:
sys.exit(returncode)
else:
_check_all(cmd, linted_filenames)
exclude_globs.append(line.strip())
formatted_filenames = []
for path in lintutils.get_sources(arguments.source_dir, exclude_globs):
formatted_filenames.append(str(path))
if arguments.fix:
if not arguments.quiet:
print("\n".join(map(lambda x: "Formatting {}".format(x),
formatted_filenames)))
# Break clang-format invocations into chunks: each invocation formats
# 16 files. Wait for all processes to complete
results = lintutils.run_parallel([
[arguments.clang_format_binary, "-i"] + some
for some in lintutils.chunk(formatted_filenames, 16)
])
for returncode, stdout, stderr in results:
# if any clang-format reported a parse error, bubble it
if returncode != 0:
sys.exit(returncode)
else:
# run an instance of clang-format for each source file in parallel,
# then wait for all processes to complete
results = lintutils.run_parallel([
[arguments.clang_format_binary, filename]
for filename in formatted_filenames
], stdout=PIPE, stderr=PIPE)
for returncode, stdout, stderr in results:
# if any clang-format reported a parse error, bubble it
help="If specified, will attempt to fix the "
"source code instead of recommending fixes, "
"defaults to %(default)s")
parser.add_argument("--quiet", default=False,
action="store_true",
help="If specified, only print errors")
arguments = parser.parse_args()
linted_filenames = []
for path in lintutils.get_sources(arguments.source_dir):
linted_filenames.append(path)
if not arguments.quiet:
msg = 'Tidying {}' if arguments.fix else 'Checking {}'
print("\n".join(map(msg.format, linted_filenames)))
cmd = [
arguments.clang_tidy_binary,
'-p',
arguments.compile_commands
]
if arguments.fix:
cmd.append('-fix')
results = lintutils.run_parallel(
[cmd + some for some in lintutils.chunk(linted_filenames, 16)])
for result in results:
result.check_returncode()
else:
_check_all(cmd, linted_filenames)
arguments.cpplint_binary, '--verbose=2', '--linelength=90',
'--filter=' + ','.join(_filters)
]
if (arguments.cpplint_binary.endswith('.py')
and platform.system() == 'Windows'):
# Windows doesn't support executable scripts; execute with
# sys.executable
cmd.insert(0, sys.executable)
if arguments.quiet:
cmd.append('--quiet')
else:
print("\n".join(map(lambda x: "Linting {}".format(x),
linted_filenames)))
# lint files in chunks: each invocation of cpplint will process 16 files
chunks = lintutils.chunk(linted_filenames, 16)
cmds = [cmd + some for some in chunks]
results = lintutils.run_parallel(cmds, stdout=PIPE, stderr=STDOUT)
error = False
# record completed processes (keyed by the first filename in the input
# chunk) for lookup in _check_some_files
completed_processes = {
_get_chunk_key(filenames): result
for filenames, result in zip(chunks, results)
}
checker = partial(_check_some_files, completed_processes)
pool = mp.Pool()
try:
# scan the outputs of various cpplint invocations in parallel to
# distill a list of problematic files
