def git_get_content_at_revision(path: Path, revision: str,
cwd: Path) -> TextDocument:
"""Get unmodified text lines of a file at a Git revision
:param path: The relative path of the file in the Git repository
:param revision: The Git revision for which to get the file content, or ``WORKTREE``
to get what's on disk right now.
:param cwd: The root of the Git repository
"""
if revision == WORKTREE:
abspath = cwd / path
mtime = datetime.utcfromtimestamp(abspath.stat().st_mtime)
return TextDocument.from_str(abspath.read_text("utf-8"),
f"{mtime} +0000")
cmd = ["git", "show", f"{revision}:./{path}"]
logger.debug("[%s]$ %s", cwd, " ".join(cmd))
try:
return TextDocument.from_str(
check_output(cmd, cwd=str(cwd), encoding="utf-8"))
except CalledProcessError as exc_info:
if exc_info.returncode == 128:
# The file didn't exist at the given revision. Act as if it was an empty
# file, so all current lines appear as edited.
return TextDocument()
else:
raise
def run_black(src: Path, src_contents: TextDocument,
black_args: BlackArgs) -> TextDocument:
"""Run the black formatter for the Python source code given as a string
Return lines of the original file as well as the formatted content.
:param src: The originating file path for the source code
:param src_contents: The source code
:param black_args: Command-line arguments to send to ``black.FileMode``
"""
config = black_args.pop("config", None)
combined_args = read_black_config(src, config)
combined_args.update(black_args)
effective_args = BlackModeAttributes()
if "line_length" in combined_args:
effective_args["line_length"] = combined_args["line_length"]
if "skip_string_normalization" in combined_args:
# The ``black`` command line argument is
# ``--skip-string-normalization``, but the parameter for
# ``black.Mode`` needs to be the opposite boolean of
# ``skip-string-normalization``, hence the inverse boolean
effective_args["string_normalization"] = not combined_args[
"skip_string_normalization"]
# Override defaults and pyproject.toml settings if they've been specified
# from the command line arguments
mode = Mode(**effective_args)
return TextDocument.from_str(
format_str(src_contents.string, mode=mode),
encoding=src_contents.encoding,
override_newline=src_contents.newline,
)
def test_debug_dump(capsys):
debug_dump(
[(1, ("black",), ("chunks",))],
TextDocument.from_str("old content"),
TextDocument.from_str("new content"),
[2, 3],
)
assert capsys.readouterr().out == (
dedent(
"""\
--------------------------------------------------------------------------------
- 1 black
+ chunks
--------------------------------------------------------------------------------
"""
)
)
def test_isort_config(monkeypatch, tmpdir, line_length, settings_file, expect):
find_project_root.cache_clear()
monkeypatch.chdir(tmpdir)
(tmpdir / 'pyproject.toml').write(
dedent(f"""\
[tool.isort]
line_length = {line_length}
"""))
content = "from module import ab, cd, ef, gh, ij, kl, mn, op, qr, st, uv, wx, yz"
config = str(tmpdir / settings_file) if settings_file else None
actual = apply_isort(TextDocument.from_str(content), Path("test1.py"),
config)
assert actual.string == expect
def apply_isort(
content: TextDocument,
src: Path,
config: Optional[str] = None,
line_length: Optional[int] = None,
) -> TextDocument:
isort_args = IsortArgs()
if config:
isort_args["settings_file"] = config
else:
isort_args["settings_path"] = str(find_project_root((str(src), )))
if line_length:
isort_args["line_length"] = line_length
logger.debug("isort.code(code=..., {})".format(", ".join(
f"{k}={v!r}" for k, v in isort_args.items())))
return TextDocument.from_str(isort_code(code=content.string, **isort_args),
encoding=content.encoding)
def test_get_path_ancestry_for_file(tmpdir):
tmpdir = Path(tmpdir)
dummy = tmpdir / "dummy"
dummy.write_text("dummy")
result = list(get_path_ancestry(dummy))
assert result[-1] == tmpdir
assert result[-2] == tmpdir.parent
@pytest.mark.parametrize(
"textdocument, expect",
[
(TextDocument(), "utf-8"),
(TextDocument(encoding="utf-8"), "utf-8"),
(TextDocument(encoding="utf-16"), "utf-16"),
(TextDocument.from_str(""), "utf-8"),
(TextDocument.from_str("", encoding="utf-8"), "utf-8"),
(TextDocument.from_str("", encoding="utf-16"), "utf-16"),
(TextDocument.from_lines([]), "utf-8"),
(TextDocument.from_lines([], encoding="utf-8"), "utf-8"),
(TextDocument.from_lines([], encoding="utf-16"), "utf-16"),
],
)
def test_textdocument_set_encoding(textdocument, expect):
"""TextDocument.encoding is correct from each constructor"""
assert textdocument.encoding == expect
@pytest.mark.parametrize(
"textdocument, expect",
[
def test_diff_and_get_opcodes():
src = TextDocument.from_str(FUNCTIONS2_PY)
dst = TextDocument.from_str(FUNCTIONS2_PY_REFORMATTED)
opcodes = diff_and_get_opcodes(src, dst)
assert opcodes == EXPECT_OPCODES
def test_opcodes_to_chunks():
src = TextDocument.from_str(FUNCTIONS2_PY)
dst = TextDocument.from_str(FUNCTIONS2_PY_REFORMATTED)
chunks = list(opcodes_to_chunks(EXPECT_OPCODES, src, dst))
assert chunks == [
(1, ("def f(",), ("def f(",)),
(2, (" a,", " **kwargs,"), (" a,", " **kwargs,")),
(
4,
(") -> A:", " with cache_dir():", " if something:"),
(") -> A:", " with cache_dir():", " if something:"),
),
(
7,
(
" result = (",
" CliRunner().invoke(black.main, [str(src1), str(src2), "
'"--diff", "--check"])',
),
(
" result = CliRunner().invoke(",
' black.main, [str(src1), str(src2), "--diff", "--check"]', # noqa: E501
),
),
(
9,
(
" )",
" limited.append(-limited.pop()) # negate top",
" return A(",
" very_long_argument_name1=very_long_value_for_the_argument,",
" very_long_argument_name2=-very.long.value.for_the_argument,",
" **kwargs,",
" )",
),
(
" )",
" limited.append(-limited.pop()) # negate top",
" return A(",
" very_long_argument_name1=very_long_value_for_the_argument,",
" very_long_argument_name2=-very.long.value.for_the_argument,",
" **kwargs,",
" )",
),
),
(16, (), ("", "")),
(16, ("def g():", ' "Docstring."'), ("def g():", ' "Docstring."')),
(18, (), ("",)),
(
18,
(" def inner():", " pass"),
(" def inner():", " pass"),
),
(20, (), ("",)),
(
20,
(' print("Inner defs should breathe a little.")',),
(' print("Inner defs should breathe a little.")',),
),
(21, (), ("", "")),
(
21,
("def h():", " def inner():", " pass"),
("def h():", " def inner():", " pass"),
),
(24, (), ("",)),
(
24,
(' print("Inner defs should breathe a little.")',),
(' print("Inner defs should breathe a little.")',),
),
]
def format_edited_parts(
srcs: Iterable[Path],
revrange: RevisionRange,
enable_isort: bool,
linter_cmdlines: List[str],
black_args: BlackArgs,
) -> Generator[Tuple[Path, TextDocument, TextDocument], None, None]:
"""Black (and optional isort) formatting for chunks with edits since the last commit
1. run isort on each edited file (optional)
2. diff the given revision and worktree (optionally with isort modifications) for
all file & dir paths on the command line
3. extract line numbers in each edited to-file for changed lines
4. run black on the contents of each edited to-file
5. get a diff between the edited to-file and the reformatted content
6. convert the diff into chunks, keeping original and reformatted content for each
chunk
7. choose reformatted content for each chunk if there were any changed lines inside
the chunk in the edited to-file, or choose the chunk's original contents if no
edits were done in that chunk
8. verify that the resulting reformatted source code parses to an identical AST as
the original edited to-file
9. write the reformatted source back to the original file
10. run linter subprocesses for all edited files (11.-14. optional)
11. diff the given revision and worktree (after isort and Black reformatting) for
each file reported by a linter
12. extract line numbers in each file reported by a linter for changed lines
13. print only linter error lines which fall on changed lines
:param srcs: Directories and files to re-format
:param revrange: The Git revision against which to compare the working tree
:param enable_isort: ``True`` to also run ``isort`` first on each changed file
:param linter_cmdlines: The command line(s) for running linters on the changed
files.
:param black_args: Command-line arguments to send to ``black.FileMode``
:return: A generator which yields details about changes for each file which should
be reformatted, and skips unchanged files.
"""
git_root = get_common_root(srcs)
changed_files = git_get_modified_files(srcs, revrange, git_root)
edited_linenums_differ = EditedLinenumsDiffer(git_root, revrange)
for path_in_repo in sorted(changed_files):
src = git_root / path_in_repo
worktree_content = TextDocument.from_str(src.read_text())
# 1. run isort
if enable_isort:
edited = apply_isort(
worktree_content,
src,
black_args.get("config"),
black_args.get("line_length"),
)
else:
edited = worktree_content
max_context_lines = len(edited.lines)
for context_lines in range(max_context_lines + 1):
# 2. diff the given revision and worktree for the file
# 3. extract line numbers in the edited to-file for changed lines
edited_linenums = edited_linenums_differ.revision_vs_lines(
path_in_repo, edited, context_lines)
if enable_isort and not edited_linenums and edited == worktree_content:
logger.debug("No changes in %s after isort", src)
break
# 4. run black
formatted = run_black(src, edited, black_args)
logger.debug("Read %s lines from edited file %s",
len(edited.lines), src)
logger.debug("Black reformat resulted in %s lines",
len(formatted.lines))
# 5. get the diff between the edited and reformatted file
opcodes = diff_and_get_opcodes(edited, formatted)
# 6. convert the diff into chunks
black_chunks = list(opcodes_to_chunks(opcodes, edited, formatted))
# 7. choose reformatted content
chosen = TextDocument.from_lines(
choose_lines(black_chunks, edited_linenums))
# 8. verify
logger.debug(
"Verifying that the %s original edited lines and %s reformatted lines "
"parse into an identical abstract syntax tree",
len(edited.lines),
len(chosen.lines),
)
try:
verify_ast_unchanged(edited, chosen, black_chunks,
edited_linenums)
except NotEquivalentError:
# Diff produced misaligned chunks which couldn't be reconstructed into
# a partially re-formatted Python file which produces an identical AST.
# Try again with a larger `-U<context_lines>` option for `git diff`,
# or give up if `context_lines` is already very large.
if context_lines == max_context_lines:
raise
logger.debug(
"AST verification failed. "
"Trying again with %s lines of context for `git diff -U`",
context_lines + 1,
)
continue
else:
# 9. A re-formatted Python file which produces an identical AST was
# created successfully - write an updated file or print the diff if
# there were any changes to the original
if chosen != worktree_content:
yield src, worktree_content, chosen
break
# 10. run linter subprocesses for all edited files (11.-14. optional)
# 11. diff the given revision and worktree (after isort and Black reformatting) for
# each file reported by a linter
# 12. extract line numbers in each file reported by a linter for changed lines
# 13. print only linter error lines which fall on changed lines
for linter_cmdline in linter_cmdlines:
run_linter(linter_cmdline, git_root, changed_files, revrange)
