def test_filter_lines_many_groups(self):
lines = ['1: foo', '12: bar', '', 'Debug: info']
self.assertEqual(
[('1', 'foo'), ('12', 'bar')],
list(utils.filter_lines(lines,
r'(?P<line>\d+): (?P<info>.*)',
groups=('line', 'info'))))
self.assertEqual(
[('1', 'foo', ':'), ('12', 'bar', ':')],
list(utils.filter_lines(
lines,
r'(?P<line>\d+)(?P<separator>:) (?P<info>.*)',
groups=('line', 'info', 'separator'))))
def test_filter_lines_many_groups(self):
lines = ['1: foo', '12: bar', '', 'Debug: info']
self.assertEqual([('1', 'foo'), ('12', 'bar')],
list(
utils.filter_lines(lines,
r'(?P<line>\d+): (?P<info>.*)',
groups=('line', 'info'))))
self.assertEqual([('1', 'foo', ':'), ('12', 'bar', ':')],
list(
utils.filter_lines(
lines,
r'(?P<line>\d+)(?P<separator>:) (?P<info>.*)',
groups=('line', 'info', 'separator'))))
def test_filter_lines_group_not_defined(self):
lines = ['1: foo', '12: bar', '', 'Debug: info']
self.assertEqual(
[('1', None), ('12', None)],
list(utils.filter_lines(lines,
r'(?P<line>\d+): .*',
groups=('line', 'debug'))))
def test_filter_lines_one_group(self):
lines = ['1: foo', '12: bar', '', 'Debug: info']
self.assertEqual(
['1', '12'],
list(utils.filter_lines(lines,
r'(?P<line>\d+): .*',
groups=('line',))))
def modified_lines(filename, extra_data, commit=None):
"""Returns the lines that have been modifed for this file.
Args:
filename: the file to check.
extra_data: is the extra_data returned by modified_files. Additionally, a
value of None means that the file was not modified.
commit: the complete sha1 (40 chars) of the commit.
Returns: a list of lines that were modified, or None in case all lines are
new.
"""
if extra_data is None:
return []
if extra_data not in ('M ', ' M', 'MM'):
return None
if commit is None:
commit = '0' * 40
commit = commit.encode('utf-8')
# Split as bytes, as the output may have some non unicode characters.
blame_lines = subprocess.check_output(
['git', 'blame', commit, '--porcelain', '--',
filename]).split(os.linesep.encode('utf-8'))
modified_line_numbers = utils.filter_lines(blame_lines,
commit +
br' (?P<line>\d+) (\d+)',
groups=('line', ))
return list(map(int, modified_line_numbers))
def modified_files(root, tracked_only=False, commit=None):
"""Returns a list of files that has been modified since the last commit.
Args:
root: the root of the repository, it has to be an absolute path.
tracked_only: exclude untracked files when True.
commit: SHA1 of the commit. If None, it will get the modified files in the
working copy.
Returns: a dictionary with the modified files as keys, and additional
information as value. In this case it adds the status returned by
hg status.
"""
assert os.path.isabs(root), "Root has to be absolute, got: %s" % root
command = ["hg", "status"]
if commit:
command.append("--change=%s" % commit)
# Convert to unicode and split
status_lines = subprocess.check_output(command).decode("utf-8").split(os.linesep)
modes = ["M", "A"]
if not tracked_only:
modes.append(r"\?")
modes_str = "|".join(modes)
modified_file_status = utils.filter_lines(
status_lines, r"(?P<mode>%s) (?P<filename>.+)" % modes_str, groups=("filename", "mode")
)
return dict((os.path.join(root, filename), mode) for filename, mode in modified_file_status)
def test_filter_lines_one_group(self):
lines = ['1: foo', '12: bar', '', 'Debug: info']
self.assertEqual(['1', '12'],
list(
utils.filter_lines(lines,
r'(?P<line>\d+): .*',
groups=('line', ))))
def modified_lines(filename, extra_data, commit=None):
"""Returns the lines that have been modifed for this file.
Args:
filename: the file to check.
extra_data: is the extra_data returned by modified_files. Additionally, a
value of None means that the file was not modified.
commit: the complete sha1 (40 chars) of the commit. Note that specifying
this value will only work (100%) when commit == last_commit (with
respect to the currently checked out revision), otherwise, we could miss
some lines.
Returns: a list of lines that were modified, or None in case all lines are
new.
"""
if extra_data is None:
return []
if extra_data not in ('M ', ' M', 'MM'):
return None
if commit is None:
commit = '0' * 40
commit = commit.encode('utf-8')
# Split as bytes, as the output may have some non unicode characters.
blame_lines = subprocess.check_output(
['git', 'blame', '--porcelain', filename]).split(
os.linesep.encode('utf-8'))
modified_line_numbers = utils.filter_lines(
blame_lines, commit + br' (?P<line>\d+) (\d+)', groups=('line', ))
return list(map(int, modified_line_numbers))
def modified_lines(filename, extra_data, commit=None):
"""Returns the lines that have been modifed for this file.
Args:
filename: the file to check.
extra_data: is the extra_data returned by modified_files. Additionally, a
value of None means that the file was not modified.
commit: the complete sha1 (40 chars) of the commit. Note that specifying
this value will only work (100%) when commit == last_commit (with
respect to the currently checked out revision), otherwise, we could miss
some lines.
Returns: a list of lines that were modified, or None in case all lines are
new.
"""
if extra_data is None:
return []
if extra_data not in ('M ', ' M', 'MM'):
return None
if commit is None:
commit = '0' * 40
commit = commit.encode('utf-8')
# Split as bytes, as the output may have some non unicode characters.
blame_lines = subprocess.check_output(
['git', 'blame', '--porcelain', filename]).splitlines()
modified_line_numbers = utils.filter_lines(blame_lines,
commit +
br' (?P<line>\d+) (\d+)',
groups=('line', ))
return list(map(int, modified_line_numbers))
def test_filter_lines_group_not_defined(self):
lines = ['1: foo', '12: bar', '', 'Debug: info']
self.assertEqual([('1', None), ('12', None)],
list(
utils.filter_lines(lines,
r'(?P<line>\d+): .*',
groups=('line', 'debug'))))
def modified_files(root, tracked_only=False, commit=None):
"""Returns a list of files that has been modified since the last commit.
Args:
root: the root of the repository, it has to be an absolute path.
tracked_only: exclude untracked files when True.
commit: SHA1 of the commit. If None, it will get the modified files in the
working copy.
Returns: a dictionary with the modified files as keys, and additional
information as value. In this case it adds the status returned by
git status.
"""
assert os.path.isabs(root), "Root has to be absolute, got: %s" % root
if commit:
return _modified_files_with_commit(root, commit)
# Convert to unicode and split
status_lines = subprocess.check_output(
['git', 'status', '--porcelain',
'--untracked-files=all']).decode('utf-8').split(os.linesep)
modes = ['M ', ' M', 'A ', 'AM']
if not tracked_only:
modes.append(r'\?\?')
modes_str = '|'.join(modes)
modified_file_status = utils.filter_lines(
status_lines,
r'(?P<mode>%s) (?P<filename>.+)' % modes_str,
groups=('filename', 'mode'))
return dict((os.path.join(root, _remove_filename_quotes(filename)), mode)
for filename, mode in modified_file_status)
def modified_files(root, tracked_only=False, commit=None):
"""Returns a list of files that has been modified since the last commit.
Args:
root: the root of the repository, it has to be an absolute path.
tracked_only: exclude untracked files when True.
commit: SHA1 of the commit. If None, it will get the modified files in the
working copy.
Returns: a dictionary with the modified files as keys, and additional
information as value. In this case it adds the status returned by
git status.
"""
assert os.path.isabs(root), "Root has to be absolute, got: %s" % root
if commit:
return _modified_files_with_commit(root, commit)
# Convert to unicode and split
status_lines = subprocess.check_output(
['git', 'status', '--porcelain',
'--untracked-files=all']).decode('utf-8').split(os.linesep)
modes = ['M ', ' M', 'A ', 'AM']
if not tracked_only:
modes.append(r'\?\?')
modes_str = '|'.join(modes)
modified_file_status = utils.filter_lines(
status_lines,
r'(?P<mode>%s) (?P<filename>.+)' % modes_str,
groups=('filename', 'mode'))
return dict((os.path.join(root, _remove_filename_quotes(filename)), mode)
for filename, mode in modified_file_status)
def test_filter_lines_no_groups(self):
lines = ['a', 'b', 'c', 'ad']
self.assertEqual(lines, list(utils.filter_lines(lines, '.')))
self.assertEqual(['a', 'ad'], list(utils.filter_lines(lines, 'a')))
self.assertEqual(['ad'], list(utils.filter_lines(lines, '.d')))
self.assertEqual(['ad'], list(utils.filter_lines(lines, 'd')))
self.assertEqual([], list(utils.filter_lines(lines, '^d')))
self.assertEqual([], list(utils.filter_lines(lines, 'foo')))
def test_filter_lines_no_groups(self):
lines = ['a', 'b', 'c', 'ad']
self.assertEqual(lines, list(utils.filter_lines(lines, '.')))
self.assertEqual(['a', 'ad'], list(utils.filter_lines(lines, 'a')))
self.assertEqual(['ad'], list(utils.filter_lines(lines, '.d')))
self.assertEqual(['ad'], list(utils.filter_lines(lines, 'd')))
self.assertEqual([], list(utils.filter_lines(lines, '^d')))
self.assertEqual([], list(utils.filter_lines(lines, 'foo')))
def _modified_files_with_commit(root, commit):
# Convert to unicode and split
status_lines = subprocess.check_output(
['git', 'diff-tree', '-r', '--root', '--no-commit-id', '--name-status',
commit]).decode('utf-8').split(os.linesep)
modified_file_status = utils.filter_lines(
status_lines,
r'(?P<mode>A|M)\s(?P<filename>.+)',
groups=('filename', 'mode'))
# We need to add a space to the mode, so to be compatible with the output
# generated by modified files.
return dict((os.path.join(root, _remove_filename_quotes(filename)),
mode + ' ') for filename, mode in modified_file_status)
def _modified_files_with_commit(root, commit):
# Convert to unicode and split
status_lines = subprocess.check_output(
['git', 'diff-tree', '-r', '--root', '--no-commit-id', '--name-status',
commit]).decode('utf-8').split(os.linesep)
modified_file_status = utils.filter_lines(
status_lines,
r'(?P<mode>A|M)\s(?P<filename>.+)',
groups=('filename', 'mode'))
# We need to add a space to the mode, so to be compatible with the output
# generated by modified files.
return dict((os.path.join(root, _remove_filename_quotes(filename)),
mode + ' ') for filename, mode in modified_file_status)
def lint_command(name, program, arguments, filter_regex, cache_enabled,
filename, lines):
"""Executes a lint program and filter the output.
Executes the lint tool 'program' with arguments 'arguments' over the file
'filename' returning only those lines matching the regular expression
'filter_regex'.
Args:
name: string: the name of the linter.
program: string: lint program.
arguments: list[string]: extra arguments for the program.
filter_regex: string: regular expression to filter lines.
cache_enabled: bool: whether using cached results is enabled.
filename: string: filename to lint.
lines: list[int]|None: list of lines that we want to capture. If None,
then all lines will be captured.
Returns: dict: a dict with the extracted info from the message.
"""
output = utils.run(name, program, arguments, cache_enabled, filename)
output_lines = output.split(os.linesep)
if lines is None:
lines_regex = r'\d+'
else:
lines_regex = '|'.join(map(str, lines))
lines_regex = '(%s)' % lines_regex
groups = ('line', 'column', 'message', 'severity', 'message_id')
filtered_lines = utils.filter_lines(output_lines,
filter_regex.format(
lines=lines_regex,
filename=re.escape(filename)),
groups=groups)
result = []
for data in filtered_lines:
comment = dict(p for p in zip(groups, data) if p[1] is not None)
if 'line' in comment:
comment['line'] = int(comment['line'])
if 'column' in comment:
comment['column'] = int(comment['column'])
if 'severity' in comment:
comment['severity'] = comment['severity'].title()
result.append(comment)
return {filename: {'comments': result}}
def modified_lines(filename, extra_data, commit=None):
"""Returns the lines that have been modifed for this file.
Args:
filename: the file to check.
extra_data: is the extra_data returned by modified_files. Additionally, a
value of None means that the file was not modified.
commit: the complete sha1 (40 chars) of the commit. Note that specifying
this value will only work (100%) when commit == last_commit (with
respect to the currently checked out revision), otherwise, we could miss
some lines.
Returns: a list of lines that were modified, or None in case all lines are
new.
"""
if extra_data is None:
return []
if extra_data != 'M':
return None
command = ['hg', 'diff', '-U', '0']
if commit:
command.append('--change=%s' % commit)
command.append(filename)
# Split as bytes, as the output may have some non unicode characters.
diff_lines = subprocess.check_output(command).split(
os.linesep.encode('utf-8'))
diff_line_numbers = utils.filter_lines(
diff_lines,
br'@@ -\d+,\d+ \+(?P<start_line>\d+),(?P<lines>\d+) @@',
groups=('start_line', 'lines'))
modified_line_numbers = []
for start_line, lines in diff_line_numbers:
start_line = int(start_line)
lines = int(lines)
modified_line_numbers.extend(range(start_line, start_line + lines))
return modified_line_numbers
def modified_lines(filename, extra_data, commit=None):
"""Returns the lines that have been modifed for this file.
Args:
filename: the file to check.
extra_data: is the extra_data returned by modified_files. Additionally, a
value of None means that the file was not modified.
commit: the complete sha1 (40 chars) of the commit. Note that specifying
this value will only work (100%) when commit == last_commit (with
respect to the currently checked out revision), otherwise, we could miss
some lines.
Returns: a list of lines that were modified, or None in case all lines are
new.
"""
if extra_data is None:
return []
if extra_data != 'M':
return None
command = ['hg', 'diff', '-U', '0']
if commit:
command.append('--change=%s' % commit)
command.append(filename)
# Split as bytes, as the output may have some non unicode characters.
diff_lines = subprocess.check_output(command).split(
os.linesep.encode('utf-8'))
diff_line_numbers = utils.filter_lines(
diff_lines,
br'@@ -\d+,\d+ \+(?P<start_line>\d+),(?P<lines>\d+) @@',
groups=('start_line', 'lines'))
modified_line_numbers = []
for start_line, lines in diff_line_numbers:
start_line = int(start_line)
lines = int(lines)
modified_line_numbers.extend(range(start_line, start_line + lines))
return modified_line_numbers
def lint_command(name, program, arguments, filter_regex, filename, lines):
"""Executes a lint program and filter the output.
Executes the lint tool 'program' with arguments 'arguments' over the file
'filename' returning only those lines matching the regular expression
'filter_regex'.
Args:
name: string: the name of the linter.
program: string: lint program.
arguments: list[string]: extra arguments for the program.
filter_regex: string: regular expression to filter lines.
filename: string: filename to lint.
lines: list[int]|None: list of lines that we want to capture. If None,
then all lines will be captured.
Returns: dict: a dict with the extracted info from the message.
"""
output = utils.get_output_from_cache(name, filename)
if output is None:
call_arguments = [program] + arguments + [filename]
try:
output = subprocess.check_output(call_arguments,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
output = error.output
except OSError:
return {
filename: {
'error': [('Could not execute "%s".%sMake sure all ' +
'required programs are installed') %
(' '.join(call_arguments), os.linesep)]
}
}
output = output.decode('utf-8')
utils.save_output_in_cache(name, filename, output)
output_lines = output.split(os.linesep)
if lines is None:
lines_regex = r'\d+'
else:
lines_regex = '|'.join(map(str, lines))
lines_regex = '(%s)' % lines_regex
groups = ('line', 'column', 'message', 'severity', 'message_id')
filtered_lines = utils.filter_lines(output_lines,
filter_regex.format(lines=lines_regex,
filename=filename),
groups=groups)
result = []
for data in filtered_lines:
comment = dict(p for p in zip(groups, data) if p[1] is not None)
if 'line' in comment:
comment['line'] = int(comment['line'])
if 'column' in comment:
comment['column'] = int(comment['column'])
if 'severity' in comment:
comment['severity'] = comment['severity'].title()
result.append(comment)
return {
filename: {
'comments': result
}
}
def lint_command(name, program, arguments, fatal_exits, filter_regex, filename, lines):
"""Executes a lint program and filter the output.
Executes the lint tool 'program' with arguments 'arguments' over the file
'filename' returning only those lines matching the regular expression
'filter_regex'.
Args:
name: string: the name of the linter.
program: string: lint program.
arguments: list[string]: extra arguments for the program.
fatal_exits: list[int]: report error if linter exit code is in the list.
filter_regex: string: regular expression to filter lines.
filename: string: filename to lint.
lines: list[int]|None: list of lines that we want to capture. If None,
then all lines will be captured.
Returns: dict: a dict with the extracted info from the message.
"""
linter_hash = utils.calculate_hash(program, arguments)
output = utils.get_output_from_cache(name, linter_hash, filename)
if output is None:
call_arguments = [program] + arguments + [filename]
try:
output = subprocess.check_output(
call_arguments, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
if error.returncode in fatal_exits:
return {
filename: {
'error': [('"%s" returned error code %i.%sOutput:%s%s') %
(' '.join(call_arguments), error.returncode, os.linesep,
error.output, os.linesep)]
}
}
else:
output = error.output
except OSError:
return {
filename: {
'error': [('Could not execute "%s".%sMake sure all ' +
'required programs are installed') %
(' '.join(call_arguments), os.linesep)]
}
}
output = output.decode('utf-8')
utils.save_output_in_cache(name, linter_hash, filename, output)
output_lines = output.split(os.linesep)
if lines is None:
lines_regex = r'\d+'
else:
lines_regex = '|'.join(map(str, lines))
lines_regex = '(%s)' % lines_regex
groups = ('line', 'column', 'message', 'severity', 'message_id')
filtered_lines = utils.filter_lines(
output_lines,
filter_regex.format(lines=lines_regex, filename=re.escape(filename)),
groups=groups)
result = []
for data in filtered_lines:
comment = dict(p for p in zip(groups, data) if p[1] is not None)
if 'line' in comment:
comment['line'] = int(comment['line'])
if 'column' in comment:
comment['column'] = int(comment['column'])
if 'severity' in comment:
comment['severity'] = comment['severity'].title()
result.append(comment)
return {filename: {'comments': result}}
