def LintFiles(files, jobs=1, progress_prefix=''):
if not IsCppLintAvailable():
print(
printer.COLOUR_RED + \
("cpplint.py not found. Please ensure the depot"
" tools are installed and in your PATH. See"
" http://dev.chromium.org/developers/how-tos/install-depot-tools for"
" details.") + \
printer.NO_COLOUR)
return -1
pool = multiprocessing.Pool(jobs)
# The '.get(9999999)' is workaround to allow killing the test script with
# ctrl+C from the shell. This bug is documented at
# http://bugs.python.org/issue8296.
tasks = [(f, progress_prefix) for f in files]
# Run under a try-catch to avoid flooding the output when the script is
# interrupted from the keyboard with ctrl+C.
try:
results = pool.map_async(LintWrapper, tasks).get(9999999)
pool.close()
pool.join()
except KeyboardInterrupt:
pool.terminate()
sys.exit(1)
n_errors = sum(results)
printer.PrintOverwritableLine(progress_prefix +
'Total errors found: %d' % n_errors)
printer.EnsureNewLine()
return n_errors
def ClangFormatFiles(files, in_place=False, jobs=1, progress_prefix=''):
if not util.IsCommandAvailable('clang-format-3.6'):
print(
printer.COLOUR_RED + \
("`clang-format-3.6` not found. Please ensure it is installed "
"and in your PATH.") + \
printer.NO_COLOUR)
return -1
pool = multiprocessing.Pool(jobs)
# The '.get(9999999)' is workaround to allow killing the test script with
# ctrl+C from the shell. This bug is documented at
# http://bugs.python.org/issue8296.
tasks = [(f, in_place, progress_prefix) for f in files]
# Run under a try-catch to avoid flooding the output when the script is
# interrupted from the keyboard with ctrl+C.
try:
results = pool.map_async(ClangFormatWrapper, tasks).get(9999999)
pool.close()
pool.join()
except KeyboardInterrupt:
pool.terminate()
sys.exit(1)
rc = sum(results)
printer.PrintOverwritableLine(progress_prefix +
'%d files are incorrectly formatted.' % rc,
type=printer.LINE_TYPE_LINTER)
printer.EnsureNewLine()
return rc
def ClangFormatFiles(files,
clang_format,
in_place=False,
jobs=1,
progress_prefix=''):
if not ClangFormatIsAvailable(clang_format):
error_message = "`{}` version {}.{} not found. Please ensure it " \
"is installed, in your PATH and the correct version." \
.format(clang_format,
CLANG_FORMAT_VERSION_MAJOR,
CLANG_FORMAT_VERSION_MINOR)
print(printer.COLOUR_RED + error_message + printer.NO_COLOUR)
return -1
pool = multiprocessing.Pool(jobs)
# The '.get(9999999)' is workaround to allow killing the test script with
# ctrl+C from the shell. This bug is documented at
# http://bugs.python.org/issue8296.
tasks = [(f, clang_format, in_place, progress_prefix) for f in files]
# Run under a try-catch to avoid flooding the output when the script is
# interrupted from the keyboard with ctrl+C.
try:
results = pool.map_async(ClangFormatWrapper, tasks).get(9999999)
pool.close()
pool.join()
except KeyboardInterrupt:
pool.terminate()
sys.exit(1)
rc = sum(results)
printer.PrintOverwritableLine(progress_prefix +
'%d files are incorrectly formatted.' % rc,
type=printer.LINE_TYPE_LINTER)
printer.EnsureNewLine()
return rc
def ClangFormatFiles(files,
clang_format,
in_place=False,
jobs=1,
progress_prefix=''):
if not ClangFormatIsAvailable(clang_format):
error_message = "`{}` version {}.{} not found. Please ensure it " \
"is installed, in your PATH and the correct version." \
.format(clang_format,
CLANG_FORMAT_VERSION_MAJOR,
CLANG_FORMAT_VERSION_MINOR)
print(printer.COLOUR_RED + error_message + printer.NO_COLOUR)
return -1
queue = TestQueue(prefix=progress_prefix)
for f in files:
queue.AddTest(f,
filename=f,
clang_format=clang_format,
in_place=in_place)
rc = queue.Run(jobs, True, RunTest)
printer.PrintOverwritableLine(progress_prefix +
'%d files are incorrectly formatted.' % rc,
type=printer.LINE_TYPE_LINTER)
printer.EnsureNewLine()
return rc
def Lint(filename, progress_prefix = ''):
command = ['cpplint.py', filename]
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
outerr, _ = process.communicate()
if process.returncode == 0:
printer.PrintOverwritableLine(
progress_prefix + "Done processing %s" % filename,
type = printer.LINE_TYPE_LINTER)
return (filename, 0)
if progress_prefix:
outerr = re.sub('^', progress_prefix, outerr, flags=re.MULTILINE)
printer.Print(outerr)
# Find the number of errors in this file.
res = re.search('Total errors found: (\d+)', outerr)
if res:
n_errors_str = res.string[res.start(1):res.end(1)]
n_errors = int(n_errors_str)
else:
print("Couldn't parse cpplint.py output.")
n_errors = -1
return (filename, n_errors)
def LintFiles(files,
jobs = 1,
progress_prefix = '',
cached_results = None):
if not IsCppLintAvailable():
print(
printer.COLOUR_RED + \
("cpplint.py not found. Please ensure the depot"
" tools are installed and in your PATH. See"
" http://dev.chromium.org/developers/how-tos/install-depot-tools for"
" details.") + \
printer.NO_COLOUR)
return -1
# Filter out directories.
files = filter(os.path.isfile, files)
# Filter out files for which we have a cached correct result.
if cached_results is not None and len(cached_results) != 0:
n_input_files = len(files)
files = filter(lambda f: ShouldLint(f, cached_results), files)
n_skipped_files = n_input_files - len(files)
if n_skipped_files != 0:
printer.Print(
progress_prefix +
'Skipping %d correct files that were already processed.' %
n_skipped_files)
pool = multiprocessing.Pool(jobs)
# The '.get(9999999)' is workaround to allow killing the test script with
# ctrl+C from the shell. This bug is documented at
# http://bugs.python.org/issue8296.
tasks = [(f, progress_prefix) for f in files]
# Run under a try-catch to avoid flooding the output when the script is
# interrupted from the keyboard with ctrl+C.
try:
results = pool.map_async(LintWrapper, tasks).get(9999999)
pool.close()
pool.join()
except KeyboardInterrupt:
pool.terminate()
sys.exit(1)
n_errors = sum(map(lambda (filename, errors): errors, results))
if cached_results is not None:
for filename, errors in results:
if errors == 0:
with open(filename, 'rb') as f:
filename = os.path.realpath(filename)
file_hash = hashlib.md5(f.read()).hexdigest()
cached_results[filename] = file_hash
printer.PrintOverwritableLine(
progress_prefix + 'Total errors found: %d' % n_errors)
printer.EnsureNewLine()
return n_errors
def Lint(filename, progress_prefix=''):
command = ['cpplint.py', filename]
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# Use a lock to avoid mixing the output for different files.
with __lint_results_lock__:
# Process the output as the process is running, until it exits.
LINT_ERROR_LINE_REGEXP = re.compile('\[[1-5]\]$')
LINT_DONE_PROC_LINE_REGEXP = re.compile('Done processing')
LINT_STATUS_LINE_REGEXP = re.compile('Total errors found')
while True:
retcode = process.poll()
while True:
line = process.stderr.readline()
if line == '': break
output_line = progress_prefix + line.rstrip('\r\n')
if LINT_ERROR_LINE_REGEXP.search(line):
printer.PrintOverwritableLine(
output_line, type=printer.LINE_TYPE_LINTER)
printer.EnsureNewLine()
elif LINT_DONE_PROC_LINE_REGEXP.search(line):
printer.PrintOverwritableLine(
output_line, type=printer.LINE_TYPE_LINTER)
elif LINT_STATUS_LINE_REGEXP.search(line):
status_line = line
if retcode != None: break
if retcode == 0:
return 0
# Return the number of errors in this file.
res = re.search('\d+$', status_line)
n_errors_str = res.string[res.start():res.end()]
n_errors = int(n_errors_str)
status_line = \
progress_prefix + 'Total errors found in %s : %d' % (filename, n_errors)
printer.PrintOverwritableLine(status_line,
type=printer.LINE_TYPE_LINTER)
printer.EnsureNewLine()
return n_errors
def ClangFormat(filename, in_place=False, progress_prefix=''):
rc = 0
printer.PrintOverwritableLine('Processing %s' % filename,
type=printer.LINE_TYPE_LINTER)
cmd_format = ['clang-format-3.6', filename]
temp_file, temp_file_name = tempfile.mkstemp(prefix='clang_format_')
cmd_format_string = '$ ' + ' '.join(cmd_format) + ' > %s' % temp_file_name
p_format = subprocess.Popen(cmd_format,
stdout=temp_file,
stderr=subprocess.STDOUT)
rc += p_format.wait()
cmd_diff = ['diff', '--unified', filename, temp_file_name]
cmd_diff_string = '$ ' + ' '.join(cmd_diff)
p_diff = subprocess.Popen(cmd_diff,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if util.IsCommandAvailable('colordiff') and not is_output_redirected:
p_colordiff = subprocess.Popen(['colordiff', '--unified'],
stdin=p_diff.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, unused = p_colordiff.communicate()
else:
out, unused = p_diff.communicate()
rc += p_diff.wait()
if in_place:
cmd_format = ['clang-format-3.6', '-i', filename]
p_format = subprocess.Popen(cmd_format,
stdout=temp_file,
stderr=subprocess.STDOUT)
if rc != 0:
printer.Print('Incorrectly formatted file: ' + filename + '\n' + \
cmd_format_string + '\n' + \
cmd_diff_string + '\n' + \
out)
os.remove(temp_file_name)
return 0 if rc == 0 else 1
def LintFiles(files,
lint_args=CPP_LINTER_RULES,
jobs=1,
verbose=False,
progress_prefix=''):
lint_options = '--filter=-,+' + ',+'.join(lint_args)
pool = multiprocessing.Pool(jobs)
# The '.get(9999999)' is workaround to allow killing the test script with
# ctrl+C from the shell. This bug is documented at
# http://bugs.python.org/issue8296.
tasks = [(f, lint_options, progress_prefix, verbose) for f in files]
results = pool.map_async(LintWrapper, tasks).get(9999999)
n_errors = sum(results)
printer.PrintOverwritableLine(progress_prefix +
'Total errors found: %d' % n_errors)
printer.EnsureNewLine()
return n_errors
def RunCommand(command, environment_options=None):
# Create a copy of the environment. We do not want to pollute the environment
# of future commands run.
environment = os.environ
# Configure the environment.
# TODO: We currently pass the options as strings, so we need to parse them. We
# should instead pass them as a data structure and build the string option
# later. `environment_options` looks like `['CXX=compiler', 'OPT=val']`.
if environment_options:
for option in environment_options:
opt, val = option.split('=')
environment[opt] = val
printable_command = ''
if environment_options:
printable_command += ' '.join(environment_options) + ' '
printable_command += ' '.join(command)
printable_command_orange = \
printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR
printer.PrintOverwritableLine(printable_command_orange)
sys.stdout.flush()
# Start a process for the command.
# Interleave `stderr` and `stdout`.
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=environment)
# We want to be able to display a continuously updated 'work indicator' while
# the process is running. Since the process can hang if the `stdout` pipe is
# full, we need to pull from it regularly. We cannot do so via the
# `readline()` function because it is blocking, and would thus cause the
# indicator to not be updated properly. So use file control mechanisms
# instead.
indicator = ' (still working: %d seconds elapsed)'
# Mark the process output as non-blocking.
flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL)
fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
t_start = time.time()
t_last_indication = t_start
process_output = ''
# Keep looping as long as the process is running.
while p.poll() is None:
# Avoid polling too often.
time.sleep(0.1)
# Update the progress indicator.
t_current = time.time()
if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1):
printer.PrintOverwritableLine(printable_command_orange +
indicator % int(t_current - t_start))
sys.stdout.flush()
t_last_indication = t_current
# Pull from the process output.
while True:
try:
line = os.read(p.stdout.fileno(), 1024)
except OSError:
line = ''
break
if line == '': break
process_output += line
# The process has exited. Don't forget to retrieve the rest of its output.
out, err = p.communicate()
rc = p.poll()
process_output += out
if rc == 0:
printer.Print(printer.COLOUR_GREEN + printable_command +
printer.NO_COLOUR)
else:
printer.Print(printer.COLOUR_RED + printable_command +
printer.NO_COLOUR)
printer.Print(process_output)
return rc
