def run_black(src: Path, src_contents: str, black_args: BlackArgs) -> List[str]:
"""Run the black formatter for the Python source code given as a string
Return lines of the original file as well as the formatted content.
:param src: The originating file path for the source code
:param src_contents: The source code as a string
:param black_args: Command-line arguments to send to ``black.FileMode``
"""
config = black_args.pop("config", None)
combined_args = read_black_config(src, config)
combined_args.update(black_args)
effective_args = BlackModeAttributes()
if "line_length" in combined_args:
effective_args["line_length"] = combined_args["line_length"]
if "skip_string_normalization" in combined_args:
# The ``black`` command line argument is
# ``--skip-string-normalization``, but the parameter for
# ``black.Mode`` needs to be the opposite boolean of
# ``skip-string-normalization``, hence the inverse boolean
effective_args["string_normalization"] = not combined_args[
"skip_string_normalization"
]
# Override defaults and pyproject.toml settings if they've been specified
# from the command line arguments
mode = Mode(**effective_args)
dst_contents = format_str(src_contents, mode=mode)
dst_lines: List[str] = dst_contents.splitlines()
return dst_lines
def file(node, filename, mode="a", skip_black=False):
"""
Convert AST to a file
:param node: AST node
:type node: ```Union[Module, ClassDef, FunctionDef]```
:param filename: emit to this file
:type filename: ```str```
:param mode: Mode to open the file in, defaults to append
:type mode: ```str```
:param skip_black: Skip formatting with black
:type skip_black: ```bool```
:return: None
:rtype: ```NoneType```
"""
if isinstance(node, (ClassDef, FunctionDef)):
node = Module(body=[node], type_ignores=[], stmt=None)
src = to_code(node)
if not skip_black:
src = format_str(
src,
mode=Mode(
target_versions=set(),
line_length=119,
is_pyi=False,
string_normalization=False,
),
)
with open(filename, mode) as f:
f.write(src)
def read_black_config(path: Optional[PathLike]) -> Mode:
"""Parse Black configuration from provided toml."""
black_mode = Mode(line_length=DEFAULT_LINE_LENGTH)
if path is None:
return black_mode
if not Path(path).is_file():
raise FileNotFoundError(f"{path} is not a file.")
try:
pyproject_toml = toml.load(path)
config = pyproject_toml.get("tool", {}).get("black", {})
except toml.TomlDecodeError as error:
raise MalformattedToml(error)
valid_black_filemode_params = sorted(
[field.name for field in fields(Mode)])
for key, val in config.items():
# this is due to FileMode param being string_normalise, but CLI being
# skip_string_normalise - https://github.com/snakemake/snakefmt/issues/73
if key.startswith("skip"):
key = key[5:]
val = not val
key = key.replace("-", "_")
if key not in valid_black_filemode_params:
continue
setattr(black_mode, key, val)
return black_mode
def run_ast_test(test_case_instance, gen_ast, gold, skip_black=False):
"""
Compares `gen_ast` with `gold` standard
:param test_case_instance: instance of `TestCase`
:type test_case_instance: ```unittest.TestCase```
:param gen_ast: generated AST
:type gen_ast: ```Union[ast.Module, ast.ClassDef, ast.FunctionDef]```
:param skip_black: Whether to skip formatting with black. Turned off for performance, turn on for pretty debug.
:type skip_black: ```bool```
:param gold: mocked AST
:type gold: ```Union[ast.Module, ast.ClassDef, ast.FunctionDef]```
"""
if isinstance(gen_ast, str):
gen_ast = ast.parse(gen_ast).body[0]
assert gen_ast is not None, "gen_ast is None"
assert gold is not None, "gold is None"
gen_ast = deepcopy(gen_ast)
gold = deepcopy(gold)
# if reindent_docstring:
# gen_docstring = ast.get_docstring(gen_ast)
# if gen_docstring is not None:
# gen_ast.body[0] = set_value(
# "\n{}".format(indent(cleandoc(gen_docstring), tab))
# )
# gold.body[0] = set_value(
# "\n{}".format(indent(ast.get_docstring(gold, clean=True), tab))
# )
# from meta.asttools import print_ast
#
# print("#gen")
# print_ast(gen_ast)
# print("#gold")
# print_ast(gold)
test_case_instance.assertEqual(*map(
identity if skip_black else partial(
format_str,
mode=Mode(
target_versions=set(),
line_length=60,
is_pyi=False,
string_normalization=False,
),
),
map(source_transformer.to_code, (gold, gen_ast)),
))
test_case_instance.assertTrue(cmp_ast(gen_ast, gold),
"Generated AST doesn't match reference AST")
def _parse_pyproject_config(context: click.Context, param: click.Parameter,
value: Optional[str]) -> Mode:
if not value:
pyproject_toml = find_pyproject_toml(
tuple(context.params.get("files", (".", ))))
value = pyproject_toml if pyproject_toml else None
if value:
try:
pyproject_toml = toml.load(value)
config = pyproject_toml.get("tool", {}).get("docstrfmt", {})
config = {
k.replace("--", "").replace("-", "_"): v
for k, v in config.items()
}
except (OSError, ValueError) as e: # pragma: no cover
raise click.FileError(
filename=value, hint=f"Error reading configuration file: {e}")
if config:
for key in ["exclude", "extend_exclude", "files"]:
config_value = config.get(key)
if config_value is not None and not isinstance(
config_value, list):
raise click.BadOptionUsage(
key, f"Config key {key} must be a list")
params = {}
if context.params is not None:
params.update(context.params)
params.update(config)
context.params = params
black_config = parse_pyproject_toml(value)
black_config.pop("exclude", None)
black_config.pop("extend_exclude", None)
target_version = black_config.pop("target_version", ["PY37"])
if target_version:
target_version = set(
getattr(TargetVersion, version.upper())
for version in target_version
if hasattr(TargetVersion, version.upper()))
black_config["target_versions"] = target_version
return Mode(**black_config)
else:
return Mode(line_length=88)
def format_python_black(file_path, file_content, **kwargs):
from black import format_file_contents, Mode, NothingChanged, InvalidInput
try:
return format_file_contents(file_content, fast=False, mode=Mode())
except NothingChanged:
return file_content
except InvalidInput:
logger.warning("{} is not valid python code.".format(file_path))
return file_content
def run_ast_test(test_case_instance, gen_ast, gold, skip_black=False):
"""
Compares `gen_ast` with `gold` standard
:param test_case_instance: instance of `TestCase`
:type test_case_instance: ```unittest.TestCase```
:param gen_ast: generated AST
:type gen_ast: ```Union[ast.Module, ast.ClassDef, ast.FunctionDef]```
:param gold: mocked AST
:type gold: ```Union[ast.Module, ast.ClassDef, ast.FunctionDef]```
:param skip_black: Whether to skip black
:type skip_black: ```bool```
"""
if isinstance(gen_ast, str):
gen_ast = ast.parse(gen_ast).body[0]
assert gen_ast is not None, "gen_ast is None"
assert gold is not None, "gold is None"
gen_ast = deepcopy(gen_ast)
gold = deepcopy(gold)
if hasattr(gen_ast, "body") and len(gen_ast.body) > 0:
gen_docstring = ast.get_docstring(gen_ast)
gold_docstring = ast.get_docstring(gold)
if gen_docstring is not None and gold_docstring is not None:
test_case_instance.assertEqual(gold_docstring.strip(),
gen_docstring.strip())
# Following test issue with docstring indentation, remove them from the AST, as symmetry has been confirmed
gen_ast.body.pop(0)
gold.body.pop(0)
test_case_instance.assertEqual(*map(
partial(
(lambda _, **kwargs: _) if skip_black else format_str,
mode=Mode(
target_versions=set(),
line_length=60,
is_pyi=False,
string_normalization=False,
),
),
map(doctrans.source_transformer.to_code, (gold, gen_ast)),
))
# from meta.asttools import print_ast
# print_ast(gen_ast)
# print_ast(gold)
test_case_instance.assertTrue(cmp_ast(gen_ast, gold),
"Generated AST doesn't match reference AST")
def __str__(self) -> str:
code = cst.Module(body=[self.stmt]).code
if self.style:
try:
from black import Mode, format_str
except ImportError:
raise ImportError("Must install black to restyle code.")
code = format_str(code, mode=Mode())
if code.endswith("\n"):
# Strip trailing newline without stripping deliberate ones.
code = code[:-1]
return code
def black_mode():
from black import Mode, parse_pyproject_toml, target_version_option_callback
config = parse_pyproject_toml(AIRFLOW_BREEZE_SOURCES_PATH / "pyproject.toml")
target_versions = set(
target_version_option_callback(None, None, tuple(config.get('target_version', ()))),
)
return Mode(
target_versions=target_versions,
line_length=bool(config.get('line_length', Mode.line_length)),
is_pyi=bool(config.get('is_pyi', Mode.is_pyi)),
string_normalization=not bool(config.get('skip_string_normalization', not Mode.string_normalization)),
preview=bool(config.get('preview', Mode.preview)),
)
def black_mode():
from black import Mode, parse_pyproject_toml, target_version_option_callback
config = parse_pyproject_toml(
Path(get_breeze_pyproject_toml_dir(), "pyproject.toml"))
target_versions = set(
target_version_option_callback(None, None,
tuple(config.get('target_version',
()))), )
return Mode(
target_versions=target_versions,
line_length=bool(config.get('line_length', Mode.line_length)),
is_pyi=bool(config.get('is_pyi', Mode.is_pyi)),
string_normalization=not bool(
config.get('skip_string_normalization',
not Mode.string_normalization)),
experimental_string_processing=bool(
config.get('experimental_string_processing',
Mode.experimental_string_processing)),
)
NothingChanged,
format_cell,
format_file_contents,
format_file_in_place,
)
import pytest
from black import Mode
from _pytest.monkeypatch import MonkeyPatch
from tests.util import DATA_DIR
pytestmark = pytest.mark.jupyter
pytest.importorskip("IPython", reason="IPython is an optional dependency")
pytest.importorskip("tokenize_rt",
reason="tokenize-rt is an optional dependency")
JUPYTER_MODE = Mode(is_ipynb=True)
EMPTY_CONFIG = DATA_DIR / "empty_pyproject.toml"
runner = CliRunner()
def test_noop() -> None:
src = 'foo = "a"'
with pytest.raises(NothingChanged):
format_cell(src, fast=True, mode=JUPYTER_MODE)
@pytest.mark.parametrize("fast", [True, False])
def test_trailing_semicolon(fast: bool) -> None:
src = 'foo = "a" ;'
def black_str_formatter(str_to_format: str) -> str:
"""
The passed string is valid python that is returned with black formatting
applied.
"""
return format_str(str_to_format, mode=Mode())
def __str__(self):
try:
return format_file_contents(self._json_str, fast=True, mode=Mode())
except:
return pformat(json.loads(self._json_str), indent=2)
import ast
from contextlib import suppress
from pathlib import Path
from typing import Iterable, Tuple, Dict
import astor
from black import Mode, TargetVersion, format_file_contents, NothingChanged
BLACK_MODE = Mode(
target_versions={TargetVersion.PY36},
line_length=88, # default value
is_pyi=False, # for rendering type stubs
string_normalization=True,
experimental_string_processing=
True, # n.b. noted to occasionally cause crashes
)
def black_format(s: str):
"""Perform code formatting on unformatted source code."""
return format_file_contents(
s,
fast=False,
mode=BLACK_MODE,
)
def render_module(module: ast.Module) -> str:
"""Transform AST module into source code."""
source = astor.to_source(module)
with suppress(NothingChanged):
def main(
context: Context,
mode: Mode,
raw_input: str,
raw_output: bool,
verbose: int,
line_length: int,
file_type: str,
check: bool,
include_txt: bool,
exclude: List[str],
extend_exclude: List[str],
quiet: bool,
docstring_trailing_line: bool,
files: List[str],
) -> None:
reporter.level = verbose
if "-" in files and len(files) > 1:
reporter.error("ValueError: stdin can not be used with other paths")
context.exit(2)
if quiet or raw_output or files == ["-"]:
reporter.level = -1
misformatted_files = set()
if line_length != 88:
mode.line_length = line_length
error_count = 0
if raw_input:
manager = Manager(reporter, mode, docstring_trailing_line)
file = "<raw_input>"
check = False
try:
misformatted = False
if file_type == "py":
misformatted, _ = _process_python(check, file, raw_input,
line_length, manager, True)
elif file_type == "rst":
misformatted, _ = _process_rst(check, file, raw_input,
line_length, manager, True)
if misformatted:
misformatted_files.add(file)
except InvalidRstErrors as errors:
reporter.error(str(errors))
context.exit(1)
context.exit(0)
cache = FileCache(context)
if len(files) < 2:
for file in files:
misformatted, error_count = _format_file(
check,
Path(file),
file_type,
include_txt,
line_length,
mode,
docstring_trailing_line,
raw_output,
None,
)
if misformatted:
misformatted_files.add(file)
else:
# This code is heavily based on that of psf/black
# see here for license: https://github.com/psf/black/blob/master/LICENSE
executor = None
loop = asyncio.get_event_loop()
worker_count = os.cpu_count()
if sys.platform == "win32": # pragma: no cover
# Work around https://bugs.python.org/issue26903
worker_count = min(worker_count, 61)
try:
executor = ProcessPoolExecutor(max_workers=worker_count)
except (ImportError, OSError): # pragma: no cover
# we arrive here if the underlying system does not support multi-processing
# like in AWS Lambda or Termux, in which case we gracefully fallback to
# a ThreadPollExecutor with just a single worker (more workers would not do us
# any good due to the Global Interpreter Lock)
executor = ThreadPoolExecutor(max_workers=1)
try:
misformatted_files, error_count = loop.run_until_complete(
_run_formatter(
check,
file_type,
files,
include_txt,
docstring_trailing_line,
mode,
raw_output,
cache,
loop,
executor,
))
finally:
shutdown(loop)
if executor is not None:
executor.shutdown()
if misformatted_files and not raw_output:
if check:
reporter.print(
f"{len(misformatted_files)} out of {plural('file', len(files))} could be reformatted."
)
else:
reporter.print(
f"{len(misformatted_files)} out of {plural('file', len(files))} were reformatted."
)
elif not raw_output:
reporter.print(f"{plural('file', len(files))} were checked.")
if error_count > 0:
reporter.print(
f"Done, but {plural('error', error_count)} occurred ❌💥❌")
elif not raw_output:
reporter.print("Done! 🎉")
if (check and misformatted_files) or error_count:
context.exit(1)
context.exit(0)