def sync(self):
"""Sync the latest state of the repository."""
if not os.path.exists(self.repo_path):
os.makedirs(self.repo_path)
logging.info('Cloning repository %s to %s', self.clone_path,
self.repo_path)
execute(['hg', 'clone', '-U', self.clone_path, self.repo_path])
else:
logging.info('Pulling into existing repository %s', self.repo_path)
execute(['hg', '-R', self.repo_path, 'pull'])
def sync(self):
"""Sync the latest state of the repository."""
if not os.path.exists(self.repo_path):
os.makedirs(self.repo_path)
logging.info('Cloning repository %s to %s',
self.clone_path, self.repo_path)
execute(['hg', 'clone', '-U', self.clone_path,
self.repo_path])
else:
logging.info('Pulling into existing repository %s',
self.repo_path)
execute(['hg', '-R', self.repo_path, 'pull'])
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
file_ext = settings['file_ext'].strip()
if file_ext:
ext = splitext(f.dest_file)[1][1:]
if not ext.lower() in file_ext.split(','):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
rulesets = settings['rulesets']
if rulesets.startswith('<?xml'):
rulesets = make_tempfile(rulesets)
outfile = make_tempfile()
execute(
[
config['pmd_path'],
'pmd',
'-d', path,
'-R', rulesets,
'-f', 'csv',
'-r', outfile
],
ignore_errors=True)
with open(outfile) as result:
reader = csv.DictReader(result)
for row in reader:
try:
f.comment(row['Description'], int(row['Line']))
except Exception as e:
logging.error('Cannot parse line "%s": %s', row, e)
def sync(self):
"""Sync the latest state of the repository."""
if not os.path.exists(self.repo_path):
os.makedirs(self.repo_path)
logging.info('Cloning repository %s to %s',
self.clone_path, self.repo_path)
execute(['git', 'clone', '--bare', self.clone_path,
self.repo_path])
else:
logging.info('Fetching into existing repository %s',
self.repo_path)
execute(['git', '--git-dir=%s' % self.repo_path, 'fetch',
'origin', '+refs/heads/*:refs/heads/*', '--prune'])
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not f.dest_file.lower().endswith('.py'):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
self.output = execute(
[
'pydocstyle',
'--ignore=%s' % settings['ignore'],
path,
],
ignore_errors=True)
for line in filter(None, self.output.split(path + ':')):
try:
line_num, message = line.split(':', 1)
line_num = line_num.split()
f.comment(message.strip(), int(line_num[0]))
except Exception:
pass
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not f.dest_file.endswith('.py'):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
output = execute([
'pyflakes',
path,
],
split_lines=True,
ignore_errors=True)
for line in output:
parsed = line.split(':', 2)
lnum = int(parsed[1])
msg = parsed[2]
f.comment('%s' % (msg, ), lnum)
def execute(self,
review,
settings={},
repository=None,
base_commit_id=None):
"""Perform a review using the tool.
Args:
review (reviewbot.processing.review.Review):
The review object.
settings (dict):
Tool-specific settings.
repository (reviewbot.repositories.Repository):
The repository.
base_commit_id (unicode):
The ID of the commit that the patch should be applied to.
"""
cmd = [
'buildbot',
'try',
'--wait',
'--quiet',
'--diff=%s' % review.get_patch_file_path(),
'--patchlevel=1',
'--username=%s' % settings['username'],
'--master=%s:%s' % (settings['address'], settings['port']),
]
branch = review.api_root.get_review_request(
review_request_id=review.request_id).branch
if branch != '' and settings['use_branch']:
cmd.append('--branch=%s' % branch)
elif 'default_branch' in settings:
cmd.append('--branch=%s' % settings['default_branch'])
if settings['connect_method'] == 'PB':
cmd.extend([
'--connect=pb',
'--passwd=%s' % settings['password'],
])
else:
# Assume SSH
cmd.extend([
'--connect=ssh',
'--jobdir=%s' % settings['jobdir'],
'--host=%s' % settings['address'],
])
for builder in settings['builders'].split(','):
cmd.append('--builder=%s' % builder.strip())
if settings['buildbotbin'] != '':
cmd.append('--buildbotbin=%s' % settings['buildbotbin'])
review.body_top = execute(cmd, ignore_errors=True)
def execute(self, review, settings={}, repository=None,
base_commit_id=None):
"""Perform a review using the tool.
Args:
review (reviewbot.processing.review.Review):
The review object.
settings (dict):
Tool-specific settings.
repository (reviewbot.repositories.Repository):
The repository.
base_commit_id (unicode):
The ID of the commit that the patch should be applied to.
"""
cmd = [
'buildbot',
'try',
'--wait',
'--quiet',
'--diff=%s' % review.get_patch_file_path(),
'--patchlevel=1',
'--username=%s' % settings['username'],
'--master=%s:%s' % (settings['address'],
settings['port']
),
]
branch = review.api_root.get_review_request(
review_request_id=review.request_id).branch
if branch != '' and settings['use_branch']:
cmd.append('--branch=%s' % branch)
elif 'default_branch' in settings:
cmd.append('--branch=%s' % settings['default_branch'])
if settings['connect_method'] == 'PB':
cmd.extend([
'--connect=pb',
'--passwd=%s' % settings['password'],
])
else:
# Assume SSH
cmd.extend([
'--connect=ssh',
'--jobdir=%s' % settings['jobdir'],
'--host=%s' % settings['address'],
])
for builder in settings['builders'].split(','):
cmd.append('--builder=%s' % builder.strip())
if settings['buildbotbin'] != '':
cmd.append('--buildbotbin=%s' % settings['buildbotbin'])
review.body_top = execute(cmd, ignore_errors=True)
def checkout(self, commit_id):
"""Check out the given commit.
Args:
commit_id (unicode):
The ID of the commit to check out.
Returns:
unicode:
The name of a directory with the given checkout.
"""
workdir = make_tempdir()
logging.info('Creating working tree for commit ID %s in %s', commit_id,
workdir)
execute(['hg', '-R', self.repo_path, 'archive', '-r', commit_id,
'-t', 'files', workdir])
return workdir
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not f.dest_file.lower().endswith('.java'):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
cfgXml = make_tempfile(settings['config'])
outfile = make_tempfile()
execute([
'java',
'-jar',
config['checkstyle_path'],
'-c',
cfgXml,
'-f',
'xml',
'-o',
outfile,
path,
],
ignore_errors=True)
try:
root = ElementTree.parse(outfile).getroot()
for row in root.iter('error'):
f.comment(row.get('message'), int(row.get('line')))
except Exception as e:
logging.error('Cannot parse xml file: %s', e)
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
filename = f.dest_file.lower()
if not filename.endswith(('.c', '.cpp', '.cxx', '.m', '.mm')):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
additional_args = []
configured_args = settings.get('cmdline_args')
if configured_args:
additional_args = shlex.split(configured_args)
outfile = make_tempfile()
command = ['clang', '-S', '--analyze']
if filename.endswith('.m'):
command.append('-ObjC')
elif filename.endswith('.mm'):
command.append('-ObjC++')
command += additional_args
command += [
path, '-Xanalyzer', '-analyzer-output=plist', '-o', outfile
]
self.output = execute(command, ignore_errors=True)
results = plistlib.readPlist(outfile)
for diagnostic in results['diagnostics']:
file_index = diagnostic['location']['file']
filename = results['files'][file_index]
if filename != f.dest_file:
continue
line, num_lines = self._find_linenums(diagnostic)
f.comment(diagnostic['description'], line, num_lines)
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
filename = f.dest_file.lower()
if not filename.endswith(('.c', '.cpp', '.cxx', '.m', '.mm')):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
additional_args = []
configured_args = settings.get('cmdline_args')
if configured_args:
additional_args = shlex.split(configured_args)
outfile = make_tempfile()
command = ['clang', '-S', '--analyze']
if filename.endswith('.m'):
command.append('-ObjC')
elif filename.endswith('.mm'):
command.append('-ObjC++')
command += additional_args
command += [path, '-Xanalyzer', '-analyzer-output=plist', '-o',
outfile]
self.output = execute(command, ignore_errors=True)
results = plistlib.readPlist(outfile)
for diagnostic in results['diagnostics']:
file_index = diagnostic['location']['file']
filename = results['files'][file_index]
if filename != f.dest_file:
continue
line, num_lines = self._find_linenums(diagnostic)
f.comment(diagnostic['description'], line, num_lines)
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not f.dest_file.lower().endswith('.java'):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
cfgXml = make_tempfile(settings['config'])
outfile = make_tempfile()
execute(
[
'java',
'-jar',
config['checkstyle_path'],
'-c', cfgXml,
'-f', 'xml',
'-o', outfile,
path,
],
ignore_errors=True)
try:
root = ElementTree.parse(outfile).getroot()
for row in root.iter('error'):
f.comment(row.get('message'), int(row.get('line')))
except Exception as e:
logging.error('Cannot parse xml file: %s', e)
def checkout(self, commit_id):
"""Check out the given commit.
Args:
commit_id (unicode):
The ID of the commit to check out.
Returns:
unicode:
The name of a directory with the given checkout.
"""
workdir = make_tempdir()
branchname = 'br-%s' % commit_id
logging.info('Creating temporary branch for clone in repo %s',
self.repo_path)
execute(['git', '--git-dir=%s' % self.repo_path, 'branch', branchname,
commit_id])
logging.info('Creating working tree for commit ID %s in %s', commit_id,
workdir)
execute(['git', 'clone', '--local', '--no-hardlinks', '--depth', '1',
'--branch', branchname, self.repo_path, workdir])
logging.info('Removing temporary branch for clone in repo %s',
self.repo_path)
execute(['git', '--git-dir=%s' % self.repo_path, 'branch', '-d',
branchname])
return workdir
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
path = f.get_patched_file_path()
if not path:
return
# Ignore the file if it does not have a .sh extension or a shebang
# with a supported shell.
if not f.dest_file.lower().endswith('.sh'):
with open(path) as patched_file:
first_line = patched_file.readline()
if not self.shebang_regex.search(first_line):
return
try:
output = execute([
'shellcheck',
'--severity=%s' % settings['severity'],
'--exclude=%s' % settings['exclude'],
'--format=gcc',
path,
],
split_lines=True,
ignore_errors=True)
except Exception as e:
logger.exception('ShellCheck failed: %s', e)
for line in output:
try:
# Strip off the filename, since it might have colons in it.
line = line[len(path) + 1:]
line_num, column, message = line.split(':', 2)
f.comment(message.strip(), int(line_num))
except Exception as e:
logger.exception('Cannot parse the shellcheck output: %s', e)
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not f.dest_file.lower().endswith('.py'):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
output = execute(
[
'pycodestyle',
'--max-line-length=%s' % settings['max_line_length'],
'--ignore=%s' % settings['ignore'],
path,
],
split_lines=True,
ignore_errors=True)
for line in output:
try:
# Strip off the filename, since it might have colons in it.
line = line[len(path) + 1:]
line_num, column, message = line.split(':', 2)
f.comment(message.strip(), int(line_num))
except:
pass
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
# Check if we should process this file, based on its extension.
if not (f.dest_file.endswith('.js') or
(self.file_exts and f.dest_file.endswith(self.file_exts))):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
cmd = ['jshint', '--extract=%s' % settings['extract_js_from_html']]
if settings['verbose']:
cmd.append('--verbose')
if self.config_file:
cmd.append('--config=%s' % self.config_file)
cmd.append(path)
output = execute(cmd, split_lines=True, ignore_errors=True)
for line in output:
m = re.match(self.REGEX, line)
if m:
f.comment('Col: %s\n%s' % (m.group('col'), m.group('msg')),
int(m.group('line_num')))
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
# Check if we should process this file, based on its extension.
if not (f.dest_file.endswith('.js') or
(self.file_exts and f.dest_file.endswith(self.file_exts))):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
cmd = ['jshint', '--extract=%s' % settings['extract_js_from_html']]
if settings['verbose']:
cmd.append('--verbose')
if self.config_file:
cmd.append('--config=%s' % self.config_file)
cmd.append(path)
output = execute(cmd, split_lines=True, ignore_errors=True)
for line in output:
m = re.match(self.REGEX, line)
if m:
f.comment('Col: %s\n%s' % (m.group('col'), m.group('msg')),
int(m.group('line_num')))
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not f.dest_file.lower().endswith('.rst'):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
output = execute([
'doc8',
'-q',
'--max-line-length=%s' % settings['max_line_length'],
'--file-encoding=%s' % settings['encoding'],
path,
],
split_lines=True,
ignore_errors=True)
for line in output:
try:
# Strip off the filename, since it might have colons in it.
line = line[len(path) + 1:]
line_num, message = line.split(':', 1)
f.comment(message.strip(), int(line_num))
except Exception:
logging.error('Cannot parse line with doc8: %s', line)
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not f.dest_file.lower().endswith('.rst'):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
output = execute(
[
'doc8', '-q',
'--max-line-length=%s' % settings['max_line_length'],
'--file-encoding=%s' % settings['encoding'],
path,
],
split_lines=True,
ignore_errors=True)
for line in output:
try:
# Strip off the filename, since it might have colons in it.
line = line[len(path) + 1:]
line_num, message = line.split(':', 1)
f.comment(message.strip(), int(line_num))
except Exception:
logging.error('Cannot parse line with doc8: %s', line)
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not (f.dest_file.lower().endswith('.cpp')
or f.dest_file.lower().endswith('.h')
or f.dest_file.lower().endswith('.c')):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
enabled_checks = []
# Check the options we want to pass to cppcheck.
if settings['style_checks_enabled']:
enabled_checks.append('style')
if settings['all_checks_enabled']:
enabled_checks.append('all')
# Create string to pass to cppcheck
enable_settings = '%s' % ','.join(map(str, enabled_checks))
cppcheck_args = [
'cppcheck',
'--template=\"{file}::{line}::{severity}::{id}::{message}\"',
'--enable=%s' % enable_settings,
]
lang = settings['force_language'].strip()
if lang:
cppcheck_args.append('--language=%s' % lang)
cppcheck_args.append(path)
# Run the script and capture the output.
output = execute(cppcheck_args, split_lines=True, ignore_errors=True)
# Now for each line extract the fields and add a comment to the file.
for line in output:
# filename.cpp,849,style,unusedFunction, \
# The function 'bob' is never used
# filename.cpp,638,style,unusedFunction, \
# The function 'peter' is never used
# filename.cpp,722,style,unusedFunction,
# The function 'test' is never used
parsed = line.split('::')
# If we have a useful message
if len(parsed) == 5:
# Sometimes we dont gets a linenumber (just and empty string)
# Catch this case and set line number to 0.
if parsed[1]:
linenumber = int(parsed[1])
else:
linenumber = 0
# Now extract the other options.
category = parsed[2]
sub_category = parsed[3]
freetext = parsed[4][:-1] # strip the " from the end
# If the message is that its an error then override the
# default settings and raise an Issue otherwise just
# add a comment.
if category == 'error':
f.comment('%s.\n\nCategory: %s\nSub Category: %s' %
(freetext, category, sub_category),
linenumber,
issue=True)
else:
f.comment('%s.\n\nCategory: %s\nSub Category: %s' %
(freetext, category, sub_category),
linenumber,
issue=False)
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not (f.dest_file.lower().endswith('.cpp') or
f.dest_file.lower().endswith('.h')):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
# Run the script and capture the output.
if settings['excluded_checks']:
output = execute(
[
'cpplint',
'--verbose=%s' % settings['verbosity'],
'--filter=%s' % settings['excluded_checks'],
path,
],
split_lines=True,
ignore_errors=True)
else:
output = execute(
[
'cpplint',
'--verbose=%s' % self.settings['verbosity'],
path,
],
split_lines=True,
ignore_errors=True)
# Now for each line extract the fields and add a comment to the file.
for line in output:
# Regexp to extract the fields from strings like:
# filename.cpp:126: \
# Use int16/int64/etc, rather than the C type long \
# [runtime/int] [4]
# filename.cpp:127: \
# Lines should be <= 132 characters long \
# [whitespace/line_length] [2]
# filename.cpp:129: \
# Use int16/int64/etc, rather than the C type long \
# [runtime/int] [4]
matching_obj = re.findall(r'(\S+:)(\d+:)(.+?\[)(.+?\])(.+)', line)
# pre declare all the variables so that they can be used outside
# the loop if the match (regexp search) worked.
linenumber = 0
freetext = ''
category = ''
verbosity = ''
for match in matching_obj:
# linenumber (: stripped from the end)
linenumber = int(match[1][:-1])
# freetext ( [ stripped from the end)
freetext = match[2][:-1].strip()
# category ( ] stripped from the end)
category = match[3][:-1].strip()
# verbosity (we just want the number between [])
verbosity = match[4][2:-1].strip()
f.comment('%s.\n\nError Group: %s\nVerbosity Level: %s' %
(freetext, category, verbosity), linenumber)
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not (f.dest_file.lower().endswith('.cpp') or
f.dest_file.lower().endswith('.h') or
f.dest_file.lower().endswith('.c')):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
enabled_checks = []
# Check the options we want to pass to cppcheck.
if settings['style_checks_enabled']:
enabled_checks.append('style')
if settings['all_checks_enabled']:
enabled_checks.append('all')
# Create string to pass to cppcheck
enable_settings = '%s' % ','.join(map(str, enabled_checks))
cppcheck_args = [
'cppcheck',
'--template=\"{file}::{line}::{severity}::{id}::{message}\"',
'--enable=%s' % enable_settings,
]
lang = settings['force_language'].strip()
if lang:
cppcheck_args.append('--language=%s' % lang)
cppcheck_args.append(path)
# Run the script and capture the output.
output = execute(cppcheck_args, split_lines=True, ignore_errors=True)
# Now for each line extract the fields and add a comment to the file.
for line in output:
# filename.cpp,849,style,unusedFunction, \
# The function 'bob' is never used
# filename.cpp,638,style,unusedFunction, \
# The function 'peter' is never used
# filename.cpp,722,style,unusedFunction,
# The function 'test' is never used
parsed = line.split('::')
# If we have a useful message
if len(parsed) == 5:
# Sometimes we dont gets a linenumber (just and empty string)
# Catch this case and set line number to 0.
if parsed[1]:
linenumber = int(parsed[1])
else:
linenumber = 0
# Now extract the other options.
category = parsed[2]
sub_category = parsed[3]
freetext = parsed[4][:-1] # strip the " from the end
# If the message is that its an error then override the
# default settings and raise an Issue otherwise just
# add a comment.
if category == 'error':
f.comment('%s.\n\nCategory: %s\nSub Category: %s' %
(freetext, category, sub_category),
linenumber, issue=True)
else:
f.comment('%s.\n\nCategory: %s\nSub Category: %s' %
(freetext, category, sub_category),
linenumber, issue=False)
def handle_file(self, f, settings):
"""Perform a review of a single file.
Args:
f (reviewbot.processing.review.File):
The file to process.
settings (dict):
Tool-specific settings.
"""
if not (f.dest_file.lower().endswith('.cpp')
or f.dest_file.lower().endswith('.h')):
# Ignore the file.
return
path = f.get_patched_file_path()
if not path:
return
# Run the script and capture the output.
if settings['excluded_checks']:
output = execute([
'cpplint',
'--verbose=%s' % settings['verbosity'],
'--filter=%s' % settings['excluded_checks'],
path,
],
split_lines=True,
ignore_errors=True)
else:
output = execute([
'cpplint',
'--verbose=%s' % self.settings['verbosity'],
path,
],
split_lines=True,
ignore_errors=True)
# Now for each line extract the fields and add a comment to the file.
for line in output:
# Regexp to extract the fields from strings like:
# filename.cpp:126: \
# Use int16/int64/etc, rather than the C type long \
# [runtime/int] [4]
# filename.cpp:127: \
# Lines should be <= 132 characters long \
# [whitespace/line_length] [2]
# filename.cpp:129: \
# Use int16/int64/etc, rather than the C type long \
# [runtime/int] [4]
matching_obj = re.findall(r'(\S+:)(\d+:)(.+?\[)(.+?\])(.+)', line)
# pre declare all the variables so that they can be used outside
# the loop if the match (regexp search) worked.
linenumber = 0
freetext = ''
category = ''
verbosity = ''
for match in matching_obj:
# linenumber (: stripped from the end)
linenumber = int(match[1][:-1])
# freetext ( [ stripped from the end)
freetext = match[2][:-1].strip()
# category ( ] stripped from the end)
category = match[3][:-1].strip()
# verbosity (we just want the number between [])
verbosity = match[4][2:-1].strip()
f.comment(
'%s.\n\nError Group: %s\nVerbosity Level: %s' %
(freetext, category, verbosity), linenumber)
