def _check_all(cmd, all_filenames): # Output from clang-tidy can be massive, so only run 16 instances # of it at a time. Process and drop that output before running more for filenames in lintutils.chunk(all_filenames, 16**2): # each clang-tidy instance will process 16 files chunks = list(lintutils.chunk(filenames, 16)) cmds = [cmd + some for some in chunks] results = lintutils.run_parallel(cmds, stderr=DEVNULL, stdout=PIPE) error = False # record completed processes (keyed by the first filename in the input chunk) # for lookup in _check_some_files completed_processes = { _get_chunk_key(some): result for some, result in zip(chunks, results) } checker = partial(_check_some_files, completed_processes) pool = mp.Pool() try: # check output of completed clang-tidy invocations in parallel for problem_files, stdout in pool.imap(checker, chunks): if problem_files: msg = "clang-tidy suggested fixes for {}" print("\n".join(map(msg.format, problem_files))) error = True except Exception: error = True raise finally: pool.terminate() pool.join() if error: sys.exit(1)
def _check_all(cmd, filenames): # each clang-tidy instance will process 16 files chunks = lintutils.chunk(filenames, 16) cmds = [cmd + some for some in chunks] results = lintutils.run_parallel(cmds, stderr=PIPE, stdout=PIPE) error = False # record completed processes (keyed by the first filename in the input # chunk) for lookup in _check_some_files completed_processes = { _get_chunk_key(some): result for some, result in zip(chunks, results) } checker = partial(_check_some_files, completed_processes) pool = mp.Pool() try: # check output of completed clang-tidy invocations in parallel for problem_files, stdout in pool.imap(checker, chunks): if problem_files: msg = "clang-tidy suggested fixes for {}" print("\n".join(map(msg.format, problem_files))) error = True except Exception: error = True raise finally: pool.terminate() pool.join() if error: sys.exit(1)
def _check_all(cmd, filenames): # each clang-tidy instance will process 16 files chunks = lintutils.chunk(filenames, 16) cmds = [cmd + some for some in chunks] results = lintutils.run_parallel(cmds, stderr=PIPE, stdout=PIPE) error = False # record completed processes (keyed by the first filename in the input # chunk) for lookup in _check_some_files completed_processes = { _get_chunk_key(some): result for some, result in zip(chunks, results) } checker = partial(_check_some_files, completed_processes) pool = mp.Pool() try: # check output of completed clang-tidy invocations in parallel for problem_files, stdout in pool.imap(checker, chunks): if problem_files: msg = "clang-tidy suggested fixes for {}" print("\n".join(map(msg.format, problem_files))) error = True except Exception: error = True raise finally: pool.terminate() pool.join() if error: sys.exit(1)
def _check_dir(arguments, source_dir, exclude_globs): formatted_filenames = [] for path in lintutils.get_sources(source_dir, exclude_globs): formatted_filenames.append(str(path)) if arguments.fix: if not arguments.quiet: print("\n".join( map(lambda x: "Formatting {}".format(x), formatted_filenames))) # Break clang-format invocations into chunks: each invocation formats # 16 files. Wait for all processes to complete results = lintutils.run_parallel( [[arguments.clang_format_binary, "-style=file", "-i"] + some for some in lintutils.chunk(formatted_filenames, 16)]) for returncode, stdout, stderr in results: # if any clang-format reported a parse error, bubble it if returncode != 0: sys.exit(returncode) else: # run an instance of clang-format for each source file in parallel, # then wait for all processes to complete results = lintutils.run_parallel( [[arguments.clang_format_binary, "-style=file", filename] for filename in formatted_filenames], stdout=PIPE, stderr=PIPE) checker_args = [] for filename, res in zip(formatted_filenames, results): # if any clang-format reported a parse error, bubble it returncode, stdout, stderr = res if returncode != 0: print(stderr) sys.exit(returncode) checker_args.append((filename, stdout)) error = False pool = mp.Pool() try: # check the output from each invocation of clang-format in parallel for filename, diff in pool.starmap(_check_one_file, checker_args): if not arguments.quiet: print("Checking {}".format(filename)) if diff: print("{} had clang-format style issues".format(filename)) # Print out the diff to stderr error = True # pad with a newline print(file=sys.stderr) sys.stderr.writelines(diff) except Exception: error = True raise finally: pool.terminate() pool.join() sys.exit(1 if error else 0)
def _check_all(cmd, filenames, ignore_checks): # each clang-tidy instance will process 16 files chunks = lintutils.chunk(filenames, 16) cmds = [cmd + some for some in chunks] results = lintutils.run_parallel(cmds, stderr=PIPE, stdout=PIPE) error = False # record completed processes (keyed by the first filename in the input # chunk) for lookup in _check_some_files completed_processes = { _get_chunk_key(some): result for some, result in zip(chunks, results) } checker = partial(_check_some_files, completed_processes) pool = mp.Pool() error = False try: cnt_error = 0 cnt_warning = 0 cnt_ignore = 0 # check output of completed clang-tidy invocations in parallel for problem_files, stdout in pool.imap(checker, chunks): if problem_files: msg = "clang-tidy suggested fixes for {}" print("\n".join(map(msg.format, problem_files))) # ignore thirdparty header file not found issue, such as: # error: 'fiu.h' file not found [clang-diagnostic-error] cnt_info = "" for line in stdout.splitlines(): if any([len(re.findall(check, line)) > 0 for check in ignore_checks]): cnt_info += line.replace(" error: ", " ignore: ").decode("utf-8") + "\n" else: cnt_info += line.decode("utf-8") + "\n" cnt_error += _count_key(cnt_info, " error: ") cnt_warning += _count_key(cnt_info, " warning: ") cnt_ignore += _count_key(cnt_info, " ignore: ") print(cnt_info) print("clang-tidy - error: {}, warning: {}, ignore {}". format(cnt_error, cnt_warning, cnt_ignore)) error = error or (cnt_error > 0 or cnt_warning > 0) except Exception: error = True raise finally: pool.terminate() pool.join() if error: sys.exit(1)
exclude_globs.append(line.strip()) formatted_filenames = [] for path in lintutils.get_sources(arguments.source_dir, exclude_globs): formatted_filenames.append(str(path)) if arguments.fix: if not arguments.quiet: print("\n".join( map(lambda x: "Formatting {}".format(x), formatted_filenames))) # Break clang-format invocations into chunks: each invocation formats # 16 files. Wait for all processes to complete results = lintutils.run_parallel( [[arguments.clang_format_binary, "-i"] + some for some in lintutils.chunk(formatted_filenames, 16)]) for returncode, stdout, stderr in results: # if any clang-format reported a parse error, bubble it if returncode != 0: sys.exit(returncode) else: # run an instance of clang-format for each source file in parallel, # then wait for all processes to complete results = lintutils.run_parallel( [[arguments.clang_format_binary, filename] for filename in formatted_filenames], stdout=PIPE, stderr=PIPE) for returncode, stdout, stderr in results: # if any clang-format reported a parse error, bubble it
'--linelength=90', '--filter=' + ','.join(_filters) ] if (arguments.cpplint_binary.endswith('.py') and platform.system() == 'Windows'): # Windows doesn't support executable scripts; execute with # sys.executable cmd.insert(0, sys.executable) if arguments.quiet: cmd.append('--quiet') else: print("\n".join(map(lambda x: "Linting {}".format(x), linted_filenames))) # lint files in chunks: each invocation of cpplint will process 16 files chunks = lintutils.chunk(linted_filenames, 16) cmds = [cmd + some for some in chunks] results = lintutils.run_parallel(cmds, stdout=PIPE, stderr=STDOUT) error = False # record completed processes (keyed by the first filename in the input # chunk) for lookup in _check_some_files completed_processes = { _get_chunk_key(filenames): result for filenames, result in zip(chunks, results) } checker = partial(_check_some_files, completed_processes) pool = mp.Pool() try: # scan the outputs of various cpplint invocations in parallel to # distill a list of problematic files
parser.add_argument("--quiet", default=False, action="store_true", help="If specified, only print errors") arguments = parser.parse_args() exclude_globs = [] if arguments.exclude_globs: for line in open(arguments.exclude_globs): exclude_globs.append(line.strip()) linted_filenames = [] for path in lintutils.get_sources(arguments.source_dir, exclude_globs): linted_filenames.append(path) if not arguments.quiet: msg = 'Tidying {}' if arguments.fix else 'Checking {}' print("\n".join(map(msg.format, linted_filenames))) cmd = [arguments.clang_tidy_binary, '-p', arguments.compile_commands] if arguments.fix: cmd.append('-fix') results = lintutils.run_parallel( [cmd + some for some in lintutils.chunk(linted_filenames, 16)]) for returncode, stdout, stderr in results: if returncode != 0: sys.exit(returncode) else: _check_all(cmd, linted_filenames)
exclude_globs.append(line.strip()) formatted_filenames = [] for path in lintutils.get_sources(arguments.source_dir, exclude_globs): formatted_filenames.append(str(path)) if arguments.fix: if not arguments.quiet: print("\n".join(map(lambda x: "Formatting {}".format(x), formatted_filenames))) # Break clang-format invocations into chunks: each invocation formats # 16 files. Wait for all processes to complete results = lintutils.run_parallel([ [arguments.clang_format_binary, "-i"] + some for some in lintutils.chunk(formatted_filenames, 16) ]) for returncode, stdout, stderr in results: # if any clang-format reported a parse error, bubble it if returncode != 0: sys.exit(returncode) else: # run an instance of clang-format for each source file in parallel, # then wait for all processes to complete results = lintutils.run_parallel([ [arguments.clang_format_binary, filename] for filename in formatted_filenames ], stdout=PIPE, stderr=PIPE) for returncode, stdout, stderr in results: # if any clang-format reported a parse error, bubble it
help="If specified, will attempt to fix the " "source code instead of recommending fixes, " "defaults to %(default)s") parser.add_argument("--quiet", default=False, action="store_true", help="If specified, only print errors") arguments = parser.parse_args() linted_filenames = [] for path in lintutils.get_sources(arguments.source_dir): linted_filenames.append(path) if not arguments.quiet: msg = 'Tidying {}' if arguments.fix else 'Checking {}' print("\n".join(map(msg.format, linted_filenames))) cmd = [ arguments.clang_tidy_binary, '-p', arguments.compile_commands ] if arguments.fix: cmd.append('-fix') results = lintutils.run_parallel( [cmd + some for some in lintutils.chunk(linted_filenames, 16)]) for result in results: result.check_returncode() else: _check_all(cmd, linted_filenames)
arguments.cpplint_binary, '--verbose=2', '--linelength=90', '--filter=' + ','.join(_filters) ] if (arguments.cpplint_binary.endswith('.py') and platform.system() == 'Windows'): # Windows doesn't support executable scripts; execute with # sys.executable cmd.insert(0, sys.executable) if arguments.quiet: cmd.append('--quiet') else: print("\n".join(map(lambda x: "Linting {}".format(x), linted_filenames))) # lint files in chunks: each invocation of cpplint will process 16 files chunks = lintutils.chunk(linted_filenames, 16) cmds = [cmd + some for some in chunks] results = lintutils.run_parallel(cmds, stdout=PIPE, stderr=STDOUT) error = False # record completed processes (keyed by the first filename in the input # chunk) for lookup in _check_some_files completed_processes = { _get_chunk_key(filenames): result for filenames, result in zip(chunks, results) } checker = partial(_check_some_files, completed_processes) pool = mp.Pool() try: # scan the outputs of various cpplint invocations in parallel to # distill a list of problematic files