def test_filter_lines_many_groups(self): lines = ['1: foo', '12: bar', '', 'Debug: info'] self.assertEqual( [('1', 'foo'), ('12', 'bar')], list(utils.filter_lines(lines, r'(?P<line>\d+): (?P<info>.*)', groups=('line', 'info')))) self.assertEqual( [('1', 'foo', ':'), ('12', 'bar', ':')], list(utils.filter_lines( lines, r'(?P<line>\d+)(?P<separator>:) (?P<info>.*)', groups=('line', 'info', 'separator'))))
def test_filter_lines_many_groups(self): lines = ['1: foo', '12: bar', '', 'Debug: info'] self.assertEqual([('1', 'foo'), ('12', 'bar')], list( utils.filter_lines(lines, r'(?P<line>\d+): (?P<info>.*)', groups=('line', 'info')))) self.assertEqual([('1', 'foo', ':'), ('12', 'bar', ':')], list( utils.filter_lines( lines, r'(?P<line>\d+)(?P<separator>:) (?P<info>.*)', groups=('line', 'info', 'separator'))))
def test_filter_lines_group_not_defined(self): lines = ['1: foo', '12: bar', '', 'Debug: info'] self.assertEqual( [('1', None), ('12', None)], list(utils.filter_lines(lines, r'(?P<line>\d+): .*', groups=('line', 'debug'))))
def test_filter_lines_one_group(self): lines = ['1: foo', '12: bar', '', 'Debug: info'] self.assertEqual( ['1', '12'], list(utils.filter_lines(lines, r'(?P<line>\d+): .*', groups=('line',))))
def modified_lines(filename, extra_data, commit=None): """Returns the lines that have been modifed for this file. Args: filename: the file to check. extra_data: is the extra_data returned by modified_files. Additionally, a value of None means that the file was not modified. commit: the complete sha1 (40 chars) of the commit. Returns: a list of lines that were modified, or None in case all lines are new. """ if extra_data is None: return [] if extra_data not in ('M ', ' M', 'MM'): return None if commit is None: commit = '0' * 40 commit = commit.encode('utf-8') # Split as bytes, as the output may have some non unicode characters. blame_lines = subprocess.check_output( ['git', 'blame', commit, '--porcelain', '--', filename]).split(os.linesep.encode('utf-8')) modified_line_numbers = utils.filter_lines(blame_lines, commit + br' (?P<line>\d+) (\d+)', groups=('line', )) return list(map(int, modified_line_numbers))
def modified_files(root, tracked_only=False, commit=None): """Returns a list of files that has been modified since the last commit. Args: root: the root of the repository, it has to be an absolute path. tracked_only: exclude untracked files when True. commit: SHA1 of the commit. If None, it will get the modified files in the working copy. Returns: a dictionary with the modified files as keys, and additional information as value. In this case it adds the status returned by hg status. """ assert os.path.isabs(root), "Root has to be absolute, got: %s" % root command = ["hg", "status"] if commit: command.append("--change=%s" % commit) # Convert to unicode and split status_lines = subprocess.check_output(command).decode("utf-8").split(os.linesep) modes = ["M", "A"] if not tracked_only: modes.append(r"\?") modes_str = "|".join(modes) modified_file_status = utils.filter_lines( status_lines, r"(?P<mode>%s) (?P<filename>.+)" % modes_str, groups=("filename", "mode") ) return dict((os.path.join(root, filename), mode) for filename, mode in modified_file_status)
def test_filter_lines_one_group(self): lines = ['1: foo', '12: bar', '', 'Debug: info'] self.assertEqual(['1', '12'], list( utils.filter_lines(lines, r'(?P<line>\d+): .*', groups=('line', ))))
def modified_lines(filename, extra_data, commit=None): """Returns the lines that have been modifed for this file. Args: filename: the file to check. extra_data: is the extra_data returned by modified_files. Additionally, a value of None means that the file was not modified. commit: the complete sha1 (40 chars) of the commit. Note that specifying this value will only work (100%) when commit == last_commit (with respect to the currently checked out revision), otherwise, we could miss some lines. Returns: a list of lines that were modified, or None in case all lines are new. """ if extra_data is None: return [] if extra_data not in ('M ', ' M', 'MM'): return None if commit is None: commit = '0' * 40 commit = commit.encode('utf-8') # Split as bytes, as the output may have some non unicode characters. blame_lines = subprocess.check_output( ['git', 'blame', '--porcelain', filename]).split( os.linesep.encode('utf-8')) modified_line_numbers = utils.filter_lines( blame_lines, commit + br' (?P<line>\d+) (\d+)', groups=('line', )) return list(map(int, modified_line_numbers))
def modified_lines(filename, extra_data, commit=None): """Returns the lines that have been modifed for this file. Args: filename: the file to check. extra_data: is the extra_data returned by modified_files. Additionally, a value of None means that the file was not modified. commit: the complete sha1 (40 chars) of the commit. Note that specifying this value will only work (100%) when commit == last_commit (with respect to the currently checked out revision), otherwise, we could miss some lines. Returns: a list of lines that were modified, or None in case all lines are new. """ if extra_data is None: return [] if extra_data not in ('M ', ' M', 'MM'): return None if commit is None: commit = '0' * 40 commit = commit.encode('utf-8') # Split as bytes, as the output may have some non unicode characters. blame_lines = subprocess.check_output( ['git', 'blame', '--porcelain', filename]).splitlines() modified_line_numbers = utils.filter_lines(blame_lines, commit + br' (?P<line>\d+) (\d+)', groups=('line', )) return list(map(int, modified_line_numbers))
def test_filter_lines_group_not_defined(self): lines = ['1: foo', '12: bar', '', 'Debug: info'] self.assertEqual([('1', None), ('12', None)], list( utils.filter_lines(lines, r'(?P<line>\d+): .*', groups=('line', 'debug'))))
def modified_files(root, tracked_only=False, commit=None): """Returns a list of files that has been modified since the last commit. Args: root: the root of the repository, it has to be an absolute path. tracked_only: exclude untracked files when True. commit: SHA1 of the commit. If None, it will get the modified files in the working copy. Returns: a dictionary with the modified files as keys, and additional information as value. In this case it adds the status returned by git status. """ assert os.path.isabs(root), "Root has to be absolute, got: %s" % root if commit: return _modified_files_with_commit(root, commit) # Convert to unicode and split status_lines = subprocess.check_output( ['git', 'status', '--porcelain', '--untracked-files=all']).decode('utf-8').split(os.linesep) modes = ['M ', ' M', 'A ', 'AM'] if not tracked_only: modes.append(r'\?\?') modes_str = '|'.join(modes) modified_file_status = utils.filter_lines( status_lines, r'(?P<mode>%s) (?P<filename>.+)' % modes_str, groups=('filename', 'mode')) return dict((os.path.join(root, _remove_filename_quotes(filename)), mode) for filename, mode in modified_file_status)
def test_filter_lines_no_groups(self): lines = ['a', 'b', 'c', 'ad'] self.assertEqual(lines, list(utils.filter_lines(lines, '.'))) self.assertEqual(['a', 'ad'], list(utils.filter_lines(lines, 'a'))) self.assertEqual(['ad'], list(utils.filter_lines(lines, '.d'))) self.assertEqual(['ad'], list(utils.filter_lines(lines, 'd'))) self.assertEqual([], list(utils.filter_lines(lines, '^d'))) self.assertEqual([], list(utils.filter_lines(lines, 'foo')))
def _modified_files_with_commit(root, commit): # Convert to unicode and split status_lines = subprocess.check_output( ['git', 'diff-tree', '-r', '--root', '--no-commit-id', '--name-status', commit]).decode('utf-8').split(os.linesep) modified_file_status = utils.filter_lines( status_lines, r'(?P<mode>A|M)\s(?P<filename>.+)', groups=('filename', 'mode')) # We need to add a space to the mode, so to be compatible with the output # generated by modified files. return dict((os.path.join(root, _remove_filename_quotes(filename)), mode + ' ') for filename, mode in modified_file_status)
def lint_command(name, program, arguments, filter_regex, cache_enabled, filename, lines): """Executes a lint program and filter the output. Executes the lint tool 'program' with arguments 'arguments' over the file 'filename' returning only those lines matching the regular expression 'filter_regex'. Args: name: string: the name of the linter. program: string: lint program. arguments: list[string]: extra arguments for the program. filter_regex: string: regular expression to filter lines. cache_enabled: bool: whether using cached results is enabled. filename: string: filename to lint. lines: list[int]|None: list of lines that we want to capture. If None, then all lines will be captured. Returns: dict: a dict with the extracted info from the message. """ output = utils.run(name, program, arguments, cache_enabled, filename) output_lines = output.split(os.linesep) if lines is None: lines_regex = r'\d+' else: lines_regex = '|'.join(map(str, lines)) lines_regex = '(%s)' % lines_regex groups = ('line', 'column', 'message', 'severity', 'message_id') filtered_lines = utils.filter_lines(output_lines, filter_regex.format( lines=lines_regex, filename=re.escape(filename)), groups=groups) result = [] for data in filtered_lines: comment = dict(p for p in zip(groups, data) if p[1] is not None) if 'line' in comment: comment['line'] = int(comment['line']) if 'column' in comment: comment['column'] = int(comment['column']) if 'severity' in comment: comment['severity'] = comment['severity'].title() result.append(comment) return {filename: {'comments': result}}
def modified_lines(filename, extra_data, commit=None): """Returns the lines that have been modifed for this file. Args: filename: the file to check. extra_data: is the extra_data returned by modified_files. Additionally, a value of None means that the file was not modified. commit: the complete sha1 (40 chars) of the commit. Note that specifying this value will only work (100%) when commit == last_commit (with respect to the currently checked out revision), otherwise, we could miss some lines. Returns: a list of lines that were modified, or None in case all lines are new. """ if extra_data is None: return [] if extra_data != 'M': return None command = ['hg', 'diff', '-U', '0'] if commit: command.append('--change=%s' % commit) command.append(filename) # Split as bytes, as the output may have some non unicode characters. diff_lines = subprocess.check_output(command).split( os.linesep.encode('utf-8')) diff_line_numbers = utils.filter_lines( diff_lines, br'@@ -\d+,\d+ \+(?P<start_line>\d+),(?P<lines>\d+) @@', groups=('start_line', 'lines')) modified_line_numbers = [] for start_line, lines in diff_line_numbers: start_line = int(start_line) lines = int(lines) modified_line_numbers.extend(range(start_line, start_line + lines)) return modified_line_numbers
def lint_command(name, program, arguments, filter_regex, filename, lines): """Executes a lint program and filter the output. Executes the lint tool 'program' with arguments 'arguments' over the file 'filename' returning only those lines matching the regular expression 'filter_regex'. Args: name: string: the name of the linter. program: string: lint program. arguments: list[string]: extra arguments for the program. filter_regex: string: regular expression to filter lines. filename: string: filename to lint. lines: list[int]|None: list of lines that we want to capture. If None, then all lines will be captured. Returns: dict: a dict with the extracted info from the message. """ output = utils.get_output_from_cache(name, filename) if output is None: call_arguments = [program] + arguments + [filename] try: output = subprocess.check_output(call_arguments, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as error: output = error.output except OSError: return { filename: { 'error': [('Could not execute "%s".%sMake sure all ' + 'required programs are installed') % (' '.join(call_arguments), os.linesep)] } } output = output.decode('utf-8') utils.save_output_in_cache(name, filename, output) output_lines = output.split(os.linesep) if lines is None: lines_regex = r'\d+' else: lines_regex = '|'.join(map(str, lines)) lines_regex = '(%s)' % lines_regex groups = ('line', 'column', 'message', 'severity', 'message_id') filtered_lines = utils.filter_lines(output_lines, filter_regex.format(lines=lines_regex, filename=filename), groups=groups) result = [] for data in filtered_lines: comment = dict(p for p in zip(groups, data) if p[1] is not None) if 'line' in comment: comment['line'] = int(comment['line']) if 'column' in comment: comment['column'] = int(comment['column']) if 'severity' in comment: comment['severity'] = comment['severity'].title() result.append(comment) return { filename: { 'comments': result } }
def lint_command(name, program, arguments, fatal_exits, filter_regex, filename, lines): """Executes a lint program and filter the output. Executes the lint tool 'program' with arguments 'arguments' over the file 'filename' returning only those lines matching the regular expression 'filter_regex'. Args: name: string: the name of the linter. program: string: lint program. arguments: list[string]: extra arguments for the program. fatal_exits: list[int]: report error if linter exit code is in the list. filter_regex: string: regular expression to filter lines. filename: string: filename to lint. lines: list[int]|None: list of lines that we want to capture. If None, then all lines will be captured. Returns: dict: a dict with the extracted info from the message. """ linter_hash = utils.calculate_hash(program, arguments) output = utils.get_output_from_cache(name, linter_hash, filename) if output is None: call_arguments = [program] + arguments + [filename] try: output = subprocess.check_output( call_arguments, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as error: if error.returncode in fatal_exits: return { filename: { 'error': [('"%s" returned error code %i.%sOutput:%s%s') % (' '.join(call_arguments), error.returncode, os.linesep, error.output, os.linesep)] } } else: output = error.output except OSError: return { filename: { 'error': [('Could not execute "%s".%sMake sure all ' + 'required programs are installed') % (' '.join(call_arguments), os.linesep)] } } output = output.decode('utf-8') utils.save_output_in_cache(name, linter_hash, filename, output) output_lines = output.split(os.linesep) if lines is None: lines_regex = r'\d+' else: lines_regex = '|'.join(map(str, lines)) lines_regex = '(%s)' % lines_regex groups = ('line', 'column', 'message', 'severity', 'message_id') filtered_lines = utils.filter_lines( output_lines, filter_regex.format(lines=lines_regex, filename=re.escape(filename)), groups=groups) result = [] for data in filtered_lines: comment = dict(p for p in zip(groups, data) if p[1] is not None) if 'line' in comment: comment['line'] = int(comment['line']) if 'column' in comment: comment['column'] = int(comment['column']) if 'severity' in comment: comment['severity'] = comment['severity'].title() result.append(comment) return {filename: {'comments': result}}