def LintFiles(files, jobs=1, progress_prefix=''): if not IsCppLintAvailable(): print( printer.COLOUR_RED + \ ("cpplint.py not found. Please ensure the depot" " tools are installed and in your PATH. See" " http://dev.chromium.org/developers/how-tos/install-depot-tools for" " details.") + \ printer.NO_COLOUR) return -1 pool = multiprocessing.Pool(jobs) # The '.get(9999999)' is workaround to allow killing the test script with # ctrl+C from the shell. This bug is documented at # http://bugs.python.org/issue8296. tasks = [(f, progress_prefix) for f in files] # Run under a try-catch to avoid flooding the output when the script is # interrupted from the keyboard with ctrl+C. try: results = pool.map_async(LintWrapper, tasks).get(9999999) pool.close() pool.join() except KeyboardInterrupt: pool.terminate() sys.exit(1) n_errors = sum(results) printer.PrintOverwritableLine(progress_prefix + 'Total errors found: %d' % n_errors) printer.EnsureNewLine() return n_errors
def ClangFormatFiles(files, in_place=False, jobs=1, progress_prefix=''): if not util.IsCommandAvailable('clang-format-3.6'): print( printer.COLOUR_RED + \ ("`clang-format-3.6` not found. Please ensure it is installed " "and in your PATH.") + \ printer.NO_COLOUR) return -1 pool = multiprocessing.Pool(jobs) # The '.get(9999999)' is workaround to allow killing the test script with # ctrl+C from the shell. This bug is documented at # http://bugs.python.org/issue8296. tasks = [(f, in_place, progress_prefix) for f in files] # Run under a try-catch to avoid flooding the output when the script is # interrupted from the keyboard with ctrl+C. try: results = pool.map_async(ClangFormatWrapper, tasks).get(9999999) pool.close() pool.join() except KeyboardInterrupt: pool.terminate() sys.exit(1) rc = sum(results) printer.PrintOverwritableLine(progress_prefix + '%d files are incorrectly formatted.' % rc, type=printer.LINE_TYPE_LINTER) printer.EnsureNewLine() return rc
def ClangFormatFiles(files, clang_format, in_place=False, jobs=1, progress_prefix=''): if not ClangFormatIsAvailable(clang_format): error_message = "`{}` version {}.{} not found. Please ensure it " \ "is installed, in your PATH and the correct version." \ .format(clang_format, CLANG_FORMAT_VERSION_MAJOR, CLANG_FORMAT_VERSION_MINOR) print(printer.COLOUR_RED + error_message + printer.NO_COLOUR) return -1 pool = multiprocessing.Pool(jobs) # The '.get(9999999)' is workaround to allow killing the test script with # ctrl+C from the shell. This bug is documented at # http://bugs.python.org/issue8296. tasks = [(f, clang_format, in_place, progress_prefix) for f in files] # Run under a try-catch to avoid flooding the output when the script is # interrupted from the keyboard with ctrl+C. try: results = pool.map_async(ClangFormatWrapper, tasks).get(9999999) pool.close() pool.join() except KeyboardInterrupt: pool.terminate() sys.exit(1) rc = sum(results) printer.PrintOverwritableLine(progress_prefix + '%d files are incorrectly formatted.' % rc, type=printer.LINE_TYPE_LINTER) printer.EnsureNewLine() return rc
def ClangFormatFiles(files, clang_format, in_place=False, jobs=1, progress_prefix=''): if not ClangFormatIsAvailable(clang_format): error_message = "`{}` version {}.{} not found. Please ensure it " \ "is installed, in your PATH and the correct version." \ .format(clang_format, CLANG_FORMAT_VERSION_MAJOR, CLANG_FORMAT_VERSION_MINOR) print(printer.COLOUR_RED + error_message + printer.NO_COLOUR) return -1 queue = TestQueue(prefix=progress_prefix) for f in files: queue.AddTest(f, filename=f, clang_format=clang_format, in_place=in_place) rc = queue.Run(jobs, True, RunTest) printer.PrintOverwritableLine(progress_prefix + '%d files are incorrectly formatted.' % rc, type=printer.LINE_TYPE_LINTER) printer.EnsureNewLine() return rc
def Lint(filename, progress_prefix = ''): command = ['cpplint.py', filename] process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) outerr, _ = process.communicate() if process.returncode == 0: printer.PrintOverwritableLine( progress_prefix + "Done processing %s" % filename, type = printer.LINE_TYPE_LINTER) return (filename, 0) if progress_prefix: outerr = re.sub('^', progress_prefix, outerr, flags=re.MULTILINE) printer.Print(outerr) # Find the number of errors in this file. res = re.search('Total errors found: (\d+)', outerr) if res: n_errors_str = res.string[res.start(1):res.end(1)] n_errors = int(n_errors_str) else: print("Couldn't parse cpplint.py output.") n_errors = -1 return (filename, n_errors)
def LintFiles(files, jobs = 1, progress_prefix = '', cached_results = None): if not IsCppLintAvailable(): print( printer.COLOUR_RED + \ ("cpplint.py not found. Please ensure the depot" " tools are installed and in your PATH. See" " http://dev.chromium.org/developers/how-tos/install-depot-tools for" " details.") + \ printer.NO_COLOUR) return -1 # Filter out directories. files = filter(os.path.isfile, files) # Filter out files for which we have a cached correct result. if cached_results is not None and len(cached_results) != 0: n_input_files = len(files) files = filter(lambda f: ShouldLint(f, cached_results), files) n_skipped_files = n_input_files - len(files) if n_skipped_files != 0: printer.Print( progress_prefix + 'Skipping %d correct files that were already processed.' % n_skipped_files) pool = multiprocessing.Pool(jobs) # The '.get(9999999)' is workaround to allow killing the test script with # ctrl+C from the shell. This bug is documented at # http://bugs.python.org/issue8296. tasks = [(f, progress_prefix) for f in files] # Run under a try-catch to avoid flooding the output when the script is # interrupted from the keyboard with ctrl+C. try: results = pool.map_async(LintWrapper, tasks).get(9999999) pool.close() pool.join() except KeyboardInterrupt: pool.terminate() sys.exit(1) n_errors = sum(map(lambda (filename, errors): errors, results)) if cached_results is not None: for filename, errors in results: if errors == 0: with open(filename, 'rb') as f: filename = os.path.realpath(filename) file_hash = hashlib.md5(f.read()).hexdigest() cached_results[filename] = file_hash printer.PrintOverwritableLine( progress_prefix + 'Total errors found: %d' % n_errors) printer.EnsureNewLine() return n_errors
def Lint(filename, progress_prefix=''): command = ['cpplint.py', filename] process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Use a lock to avoid mixing the output for different files. with __lint_results_lock__: # Process the output as the process is running, until it exits. LINT_ERROR_LINE_REGEXP = re.compile('\[[1-5]\]$') LINT_DONE_PROC_LINE_REGEXP = re.compile('Done processing') LINT_STATUS_LINE_REGEXP = re.compile('Total errors found') while True: retcode = process.poll() while True: line = process.stderr.readline() if line == '': break output_line = progress_prefix + line.rstrip('\r\n') if LINT_ERROR_LINE_REGEXP.search(line): printer.PrintOverwritableLine( output_line, type=printer.LINE_TYPE_LINTER) printer.EnsureNewLine() elif LINT_DONE_PROC_LINE_REGEXP.search(line): printer.PrintOverwritableLine( output_line, type=printer.LINE_TYPE_LINTER) elif LINT_STATUS_LINE_REGEXP.search(line): status_line = line if retcode != None: break if retcode == 0: return 0 # Return the number of errors in this file. res = re.search('\d+$', status_line) n_errors_str = res.string[res.start():res.end()] n_errors = int(n_errors_str) status_line = \ progress_prefix + 'Total errors found in %s : %d' % (filename, n_errors) printer.PrintOverwritableLine(status_line, type=printer.LINE_TYPE_LINTER) printer.EnsureNewLine() return n_errors
def ClangFormat(filename, in_place=False, progress_prefix=''): rc = 0 printer.PrintOverwritableLine('Processing %s' % filename, type=printer.LINE_TYPE_LINTER) cmd_format = ['clang-format-3.6', filename] temp_file, temp_file_name = tempfile.mkstemp(prefix='clang_format_') cmd_format_string = '$ ' + ' '.join(cmd_format) + ' > %s' % temp_file_name p_format = subprocess.Popen(cmd_format, stdout=temp_file, stderr=subprocess.STDOUT) rc += p_format.wait() cmd_diff = ['diff', '--unified', filename, temp_file_name] cmd_diff_string = '$ ' + ' '.join(cmd_diff) p_diff = subprocess.Popen(cmd_diff, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if util.IsCommandAvailable('colordiff') and not is_output_redirected: p_colordiff = subprocess.Popen(['colordiff', '--unified'], stdin=p_diff.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, unused = p_colordiff.communicate() else: out, unused = p_diff.communicate() rc += p_diff.wait() if in_place: cmd_format = ['clang-format-3.6', '-i', filename] p_format = subprocess.Popen(cmd_format, stdout=temp_file, stderr=subprocess.STDOUT) if rc != 0: printer.Print('Incorrectly formatted file: ' + filename + '\n' + \ cmd_format_string + '\n' + \ cmd_diff_string + '\n' + \ out) os.remove(temp_file_name) return 0 if rc == 0 else 1
def LintFiles(files, lint_args=CPP_LINTER_RULES, jobs=1, verbose=False, progress_prefix=''): lint_options = '--filter=-,+' + ',+'.join(lint_args) pool = multiprocessing.Pool(jobs) # The '.get(9999999)' is workaround to allow killing the test script with # ctrl+C from the shell. This bug is documented at # http://bugs.python.org/issue8296. tasks = [(f, lint_options, progress_prefix, verbose) for f in files] results = pool.map_async(LintWrapper, tasks).get(9999999) n_errors = sum(results) printer.PrintOverwritableLine(progress_prefix + 'Total errors found: %d' % n_errors) printer.EnsureNewLine() return n_errors
def RunCommand(command, environment_options=None): # Create a copy of the environment. We do not want to pollute the environment # of future commands run. environment = os.environ # Configure the environment. # TODO: We currently pass the options as strings, so we need to parse them. We # should instead pass them as a data structure and build the string option # later. `environment_options` looks like `['CXX=compiler', 'OPT=val']`. if environment_options: for option in environment_options: opt, val = option.split('=') environment[opt] = val printable_command = '' if environment_options: printable_command += ' '.join(environment_options) + ' ' printable_command += ' '.join(command) printable_command_orange = \ printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR printer.PrintOverwritableLine(printable_command_orange) sys.stdout.flush() # Start a process for the command. # Interleave `stderr` and `stdout`. p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=environment) # We want to be able to display a continuously updated 'work indicator' while # the process is running. Since the process can hang if the `stdout` pipe is # full, we need to pull from it regularly. We cannot do so via the # `readline()` function because it is blocking, and would thus cause the # indicator to not be updated properly. So use file control mechanisms # instead. indicator = ' (still working: %d seconds elapsed)' # Mark the process output as non-blocking. flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL) fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK) t_start = time.time() t_last_indication = t_start process_output = '' # Keep looping as long as the process is running. while p.poll() is None: # Avoid polling too often. time.sleep(0.1) # Update the progress indicator. t_current = time.time() if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1): printer.PrintOverwritableLine(printable_command_orange + indicator % int(t_current - t_start)) sys.stdout.flush() t_last_indication = t_current # Pull from the process output. while True: try: line = os.read(p.stdout.fileno(), 1024) except OSError: line = '' break if line == '': break process_output += line # The process has exited. Don't forget to retrieve the rest of its output. out, err = p.communicate() rc = p.poll() process_output += out if rc == 0: printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR) else: printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR) printer.Print(process_output) return rc