def _git_fetch_for_comparison(remote: str,
actual_branch: str,
compare_branch: str,
verbose: bool) -> prepared_env.PreparedEnv:
"""Fetches two branches including their common ancestor.
Limits the depth of the fetch to avoid unnecessary work. Scales up the
depth exponentially and tries again when the initial guess is not deep
enough.
Args:
remote: The location of the remote repository, in a format that the
git command will understand.
actual_branch: A remote branch or ref to fetch,
compare_branch: Another remote branch or ref to fetch,
verbose: When set, more progress output is produced.
Returns:
A ComparableCommits containing the commit id of the actual branch and
a the id of a commit to compare against (e.g. for when doing incremental
checks).
"""
actual_id = ''
base_id = ''
for depth in [10, 100, 1000, None]:
depth_str = '' if depth is None else '--depth={}'.format(depth)
shell_tools.run_cmd(
'git',
'fetch',
None if verbose else '--quiet',
remote,
actual_branch,
depth_str,
log_run_to_stderr=verbose)
actual_id = shell_tools.output_of('git', 'rev-parse', 'FETCH_HEAD')
shell_tools.run_cmd('git',
'fetch',
None if verbose else '--quiet',
remote,
compare_branch,
depth_str,
log_run_to_stderr=verbose)
base_id = shell_tools.output_of('git', 'rev-parse', 'FETCH_HEAD')
try:
base_id = shell_tools.output_of(
'git',
'merge-base',
actual_id,
base_id)
break
except subprocess.CalledProcessError:
# No common ancestor. We need to dig deeper.
pass
return prepared_env.PreparedEnv(None, actual_id, base_id, None, None)
def test_output_of():
assert shell_tools.output_of('true') == ''
with pytest.raises(subprocess.CalledProcessError):
_ = shell_tools.output_of('false')
assert shell_tools.output_of(['echo', 'test']) == 'test'
# filtering of the None arguments was removed. check this now fails
with pytest.raises(TypeError):
_ = shell_tools.output_of(['echo', 'test', None, 'duck'])
assert shell_tools.output_of('pwd', cwd='/tmp') in ['/tmp', '/private/tmp']
def get_changed_files(self) -> List[str]:
"""Get the files changed on one git branch vs another.
Returns:
List[str]: File paths of changed files, relative to the git repo
root.
"""
out = shell_tools.output_of('git',
'diff',
'--name-only',
self.compare_commit_id,
self.actual_commit_id,
'--',
cwd=self.destination_directory)
return [e for e in out.split('\n') if e.strip()]
def get_changed_files(self) -> List[str]:
"""Get the files changed on one git branch vs another.
Returns:
List[str]: File paths of changed files, relative to the git repo
root.
"""
out = shell_tools.output_of(
'git',
'diff',
'--name-only',
self.compare_commit_id,
self.actual_commit_id,
cwd=self.destination_directory)
return [e for e in out.split('\n') if e.strip()]
def get_incremental_uncovered_lines(abs_path: str,
base_commit: str,
actual_commit: Optional[str]
) -> List[Tuple[int, str, str]]:
"""
Uses git diff and the annotation files created by
`pytest --cov-report annotate` to find touched but uncovered lines in the
given file.
Args:
abs_path: The path of a file to look for uncovered lines in.
base_commit: Old state to diff against.
actual_commit: Current state. Use None to use local uncommitted files.
Returns:
A list of the indices, content, and reason-for-including of
'interesting' uncovered lines. An interesting uncovered line is one
involved with the diff.
"""
# Deleted files don't have any lines that need to be covered.
if not os.path.isfile(abs_path):
return []
unified_diff_lines_str = shell_tools.output_of(
'git',
'diff',
'--unified=0',
base_commit,
actual_commit,
'--',
abs_path)
unified_diff_lines = [e
for e in unified_diff_lines_str.split('\n')
if e.strip()]
touched_lines = diff_to_new_interesting_lines(unified_diff_lines)
with open(abs_path, 'r') as actual_file:
ignored_lines = determine_ignored_lines(actual_file.read())
cover_path = abs_path + ',cover'
has_cover_file = os.path.isfile(cover_path)
content_file = cover_path if has_cover_file else abs_path
with open(content_file, 'r') as annotated_coverage_file:
return [(i, fix_line_from_coverage_file(line), touched_lines[i])
for i, line in enumerate(annotated_coverage_file, start=1)
if i in touched_lines and i not in ignored_lines
if line_counts_as_uncovered(line, has_cover_file)]
def get_incremental_uncovered_lines(abs_path: str,
base_commit: str,
actual_commit: Optional[str]
) -> List[Tuple[int, str, str]]:
"""
Uses git diff and the annotation files created by
`pytest --cov-report annotate` to find touched but uncovered lines in the
given file.
Args:
abs_path: The path of a file to look for uncovered lines in.
base_commit: Old state to diff against.
actual_commit: Current state. Use None to use local uncommitted files.
Returns:
A list of the indices, content, and reason-for-including of
'interesting' uncovered lines. An interesting uncovered line is one
involved with the diff.
"""
# Deleted files don't have any lines that need to be covered.
if not os.path.isfile(abs_path):
return []
unified_diff_lines_str = shell_tools.output_of(
'git',
'diff',
'--unified=0',
base_commit,
actual_commit,
'--',
abs_path)
unified_diff_lines = [e
for e in unified_diff_lines_str.split('\n')
if e.strip()]
touched_lines = diff_to_new_interesting_lines(unified_diff_lines)
with open(abs_path, 'r') as actual_file:
ignored_lines = determine_ignored_lines(actual_file.read())
cover_path = abs_path + ',cover'
has_cover_file = os.path.isfile(cover_path)
content_file = cover_path if has_cover_file else abs_path
with open(content_file, 'r') as annotated_coverage_file:
return [(i, fix_line_from_coverage_file(line), touched_lines[i])
for i, line in enumerate(annotated_coverage_file, start=1)
if i in touched_lines and i not in ignored_lines
if line_counts_as_uncovered(line, has_cover_file)]
def test_output_of():
assert shell_tools.output_of('true') == ''
with pytest.raises(subprocess.CalledProcessError):
_ = shell_tools.output_of('false')
assert shell_tools.output_of('echo', 'test') == 'test'
assert shell_tools.output_of('pwd', cwd='/tmp') in ['/tmp', '/private/tmp']
def get_repo_root() -> str:
"""Get the root of the git repository the cwd is within."""
return shell_tools.output_of('git', 'rev-parse', '--show-toplevel')
def fetch_local_files(destination_directory: str,
verbose: bool) -> prepared_env.PreparedEnv:
"""Uses local files to create a directory for testing and comparisons.
Args:
destination_directory: The directory where the copied files should go.
verbose: When set, more progress output is produced.
Returns:
Commit ids corresponding to content to test/compare.
"""
staging_dir = destination_directory + '-staging'
try:
shutil.copytree(get_repo_root(), staging_dir)
os.chdir(staging_dir)
if verbose:
print('chdir', staging_dir, file=sys.stderr)
shell_tools.run_cmd('git',
'add',
'--all',
out=sys.stderr,
log_run_to_stderr=verbose)
shell_tools.run_cmd('git',
'commit',
'-m',
'working changes',
'--allow-empty',
'--no-gpg-sign',
None if verbose else '--quiet',
out=sys.stderr,
log_run_to_stderr=verbose)
cur_commit = shell_tools.output_of('git', 'rev-parse', 'HEAD')
os.chdir(destination_directory)
if verbose:
print('chdir', destination_directory, file=sys.stderr)
shell_tools.run_cmd('git',
'init',
None if verbose else '--quiet',
out=sys.stderr,
log_run_to_stderr=verbose)
result = _git_fetch_for_comparison(staging_dir,
cur_commit,
'master',
verbose=verbose)
finally:
shutil.rmtree(staging_dir, ignore_errors=True)
shell_tools.run_cmd('git',
'branch',
None if verbose else '--quiet',
'compare_commit',
result.compare_commit_id,
log_run_to_stderr=verbose)
shell_tools.run_cmd('git',
'checkout',
None if verbose else '--quiet',
'-b',
'actual_commit',
result.actual_commit_id,
log_run_to_stderr=verbose)
return prepared_env.PreparedEnv(
github_repo=None,
actual_commit_id=result.actual_commit_id,
compare_commit_id=result.compare_commit_id,
destination_directory=destination_directory,
virtual_env_path=None)
def test_output_of():
assert shell_tools.output_of('true') == ''
with pytest.raises(subprocess.CalledProcessError):
_ = shell_tools.output_of('false')
assert shell_tools.output_of('echo', 'test') == 'test'
assert shell_tools.output_of('pwd', cwd='/tmp') in ['/tmp', '/private/tmp']
def get_repo_root() -> str:
"""Get the root of the git repository the cwd is within."""
return shell_tools.output_of('git', 'rev-parse', '--show-toplevel')
def fetch_local_files(destination_directory: str,
verbose: bool) -> prepared_env.PreparedEnv:
"""Uses local files to create a directory for testing and comparisons.
Args:
destination_directory: The directory where the copied files should go.
verbose: When set, more progress output is produced.
Returns:
Commit ids corresponding to content to test/compare.
"""
staging_dir = destination_directory + '-staging'
try:
shutil.copytree(get_repo_root(), staging_dir)
os.chdir(staging_dir)
if verbose:
print('chdir', staging_dir, file=sys.stderr)
shell_tools.run_cmd(
'git',
'add',
'--all',
out=sys.stderr,
log_run_to_stderr=verbose)
shell_tools.run_cmd(
'git',
'commit',
'-m', 'working changes',
'--allow-empty',
None if verbose else '--quiet',
out=sys.stderr,
log_run_to_stderr=verbose)
cur_commit = shell_tools.output_of('git', 'rev-parse', 'HEAD')
os.chdir(destination_directory)
if verbose:
print('chdir', destination_directory, file=sys.stderr)
shell_tools.run_cmd('git',
'init',
None if verbose else '--quiet',
out=sys.stderr,
log_run_to_stderr=verbose)
result = _git_fetch_for_comparison(staging_dir,
cur_commit,
'master',
verbose=verbose)
finally:
shutil.rmtree(staging_dir, ignore_errors=True)
shell_tools.run_cmd(
'git',
'branch',
None if verbose else '--quiet',
'compare_commit',
result.compare_commit_id,
log_run_to_stderr=verbose)
shell_tools.run_cmd(
'git',
'checkout',
None if verbose else '--quiet',
'-b',
'actual_commit',
result.actual_commit_id,
log_run_to_stderr=verbose)
return prepared_env.PreparedEnv(
github_repo=None,
actual_commit_id=result.actual_commit_id,
compare_commit_id=result.compare_commit_id,
destination_directory=destination_directory,
virtual_env_path=None)
