def run(self) -> bool:
"""
Run the reformatter.
:return: Whether the file was changed.
"""
quote_formatted_code = reformat_quotes(self._unformatted_source)
yapfed_code = FormatCode(quote_formatted_code,
style_config=self.yapf_style)[0]
generic_formatted_code = reformat_generics(yapfed_code)
# TODO: support spaces
try:
isorted_code = StringList(
isort.code(generic_formatted_code, config=self.isort_config))
except FileSkipComment:
isorted_code = StringList(generic_formatted_code)
isorted_code.blankline(ensure_single=True)
self._reformatted_source = str(isorted_code)
# Fix for noqa comments being pushed to new line
self._reformatted_source = noqa_reformat(self._reformatted_source)
return self._reformatted_source != self._unformatted_source
def append_doctring_from_another(target: Union[Type, Callable],
original: Union[Type, Callable]):
"""
Sets the docstring of the ``target`` function to that of the ``original`` function.
This may be useful for subclasses or wrappers that use the same arguments.
Any indentation in either docstring is removed to
ensure consistent indentation between the two docstrings.
Bear this in mind if additional indentation is used in the docstring.
:param target: The object to append the docstring to
:param original: The object to copy the docstring from
"""
# this package
from domdf_python_tools.stringlist import StringList
target_doc = target.__doc__
original_doc = original.__doc__
if isinstance(original_doc, str) and isinstance(target_doc, str):
docstring = StringList(cleandoc(target_doc))
docstring.blankline(ensure_single=True)
docstring.append(cleandoc(original_doc))
docstring.blankline(ensure_single=True)
target.__doc__ = str(docstring)
elif not isinstance(target_doc, str) and isinstance(original_doc, str):
docstring = StringList(cleandoc(original_doc))
docstring.blankline(ensure_single=True)
target.__doc__ = str(docstring)
def make_mypy(self) -> PathPlus:
"""
Create, update or remove the mypy action, as appropriate.
.. versionadded:: 2020.1.27
"""
ci_file = self.workflows_dir / "mypy.yml"
template = self.templates.get_template(ci_file.name)
# TODO: handle case where Linux is not a supported platform
platforms = set(self.templates.globals["platforms"])
if "macOS" in platforms:
platforms.remove("macOS")
platforms = set(
filter(None, (platform_ci_names.get(p, None) for p in platforms)))
dependency_lines = self.get_linux_mypy_requirements()
linux_platform = platform_ci_names["Linux"]
if dependency_lines == self.standard_python_install_lines:
dependencies_block = StringList([
"- name: Install dependencies 🔧",
" run: |",
])
with dependencies_block.with_indent(" ", 2):
dependencies_block.extend(self.standard_python_install_lines)
else:
dependencies_block = StringList([
"- name: Install dependencies (Linux) 🔧",
f" if: ${{{{ matrix.os == '{linux_platform}' && steps.changes.outputs.code == 'true' }}}}",
" run: |",
])
with dependencies_block.with_indent(" ", 2):
dependencies_block.extend(dependency_lines)
if self.templates.globals["platforms"] != ["Linux"]:
dependencies_block.blankline(ensure_single=True)
dependencies_block.extend([
"- name: Install dependencies (Win/mac) 🔧",
f" if: ${{{{ matrix.os != '{linux_platform}' && steps.changes.outputs.code == 'true' }}}}",
" run: |",
])
with dependencies_block.with_indent(" ", 2):
dependencies_block.extend(
self.standard_python_install_lines)
ci_file.write_clean(
template.render(
platforms=sorted(platforms),
linux_platform=platform_ci_names["Linux"],
dependencies_block=indent(str(dependencies_block), " "),
code_file_filter=self._code_file_filter,
))
return ci_file
def test_sort(self):
sl = StringList(['', '', "hello", "world", '', '', '', "1234"])
sl.sort()
assert sl == ['', '', '', '', '', "1234", "hello", "world"]
assert isinstance(sl, StringList)
sl = StringList(['', '', "hello", "world", '', '', '', "1234"])
sl.sort(reverse=True)
assert sl == ["world", "hello", "1234", '', '', '', '', '']
assert isinstance(sl, StringList)
def create_body_overloads(self) -> StringList:
"""
Create the overloaded implementations for insertion into to the body of the documenter's output.
"""
output = StringList()
formatted_overloads = []
output.blankline()
# output.append(":Overloaded Implementations:")
output.append(":Overloads:")
output.blankline()
# Size varies depending on docutils config
output.indent_type = ' '
output.indent_size = self.env.app.config.docutils_tab_width # type: ignore
if self.analyzer and '.'.join(self.objpath) in self.analyzer.overloads:
for overload in self.analyzer.overloads.get('.'.join(self.objpath)): # type: ignore
overload = self.process_overload_signature(overload)
buf = [format_annotation(self.object), r"\("]
for name, param in overload.parameters.items():
buf.append(f"**{name}**")
if param.annotation is not Parameter.empty:
buf.append(r"\: ")
buf.append(format_annotation(param.annotation))
if param.default is not Parameter.empty:
buf.append(" = ")
buf.append(param.default)
buf.append(r"\, ")
if buf[-2][-1] != '`':
buf[-1] = r" )"
else:
buf[-1] = r")"
if overload.return_annotation is not Parameter.empty:
buf.append(" -> ")
buf.append(format_annotation(overload.return_annotation))
formatted_overloads.append(''.join(buf))
if len(formatted_overloads) == 1:
output.append(formatted_overloads[0])
else:
for line in formatted_overloads:
output.append(f"* {line}")
output.blankline(ensure_single=True)
return output
return StringList()
def check(self, page: BeautifulSoup, **kwargs): # type: ignore
r"""
Check an HTML page generated by Sphinx for regressions,
using `pytest-regressions <https://pypi.org/project/pytest-regressions/>`_
:param page: The page to test.
:param \*\*kwargs: Additional keyword arguments passed to
:meth:`pytest_regressions.file_regression.FileRegressionFixture.check`.
**Example usage**
.. code-block:: python
@pytest.mark.parametrize("page", ["index.html"], indirect=True)
def test_page(page: BeautifulSoup, html_regression: HTMLRegressionFixture):
html_regression.check(page, file_regression)
""" # noqa D400
__tracebackhide__ = True
page = remove_html_footer(page)
page = remove_html_link_tags(page)
for div in page.select("script"):
if "_static/language_data.js" in str(div):
div.extract()
kwargs.pop("encoding", None)
kwargs.pop("extension", None)
super().check(
str(StringList(page.prettify())),
encoding="UTF-8",
extension=".html",
)
def check_html_regression(page: BeautifulSoup,
file_regression: FileRegressionFixture):
"""
Check an HTML page generated by Sphinx for regressions,
using `pytest-regressions <https://pypi.org/project/pytest-regressions/>`_
:param page: The page to test.
:param file_regression: The file regression fixture.
**Example usage**
.. code-block:: python
@pytest.mark.parametrize("page", ["index.html"], indirect=True)
def test_page(page: BeautifulSoup, file_regression: FileRegressionFixture):
check_html_regression(page, file_regression)
""" # noqa D400
__tracebackhide__ = True
page = remove_html_footer(page)
page = remove_html_link_tags(page)
for div in page.select("script"):
if "_static/language_data.js" in str(div):
div.extract()
check_file_regression(
StringList(page.prettify()),
file_regression,
extension=".html",
)
def configure(app: Sphinx, config: Config):
"""
Configure :mod:`sphinx_toolbox.code`.
.. versionadded:: 2.11.0
:param app: The Sphinx application.
:param config:
"""
latex_elements = getattr(app.config, "latex_elements", {})
latex_preamble = StringList(latex_elements.get("preamble", ''))
latex_preamble.blankline()
latex_preamble.append(r"\definecolor{regex_literal}{HTML}{696969}")
latex_preamble.append(r"\definecolor{regex_at}{HTML}{FF4500}")
latex_preamble.append(r"\definecolor{regex_repeat_brace}{HTML}{FF4500}")
latex_preamble.append(r"\definecolor{regex_branch}{HTML}{FF4500}")
latex_preamble.append(r"\definecolor{regex_subpattern}{HTML}{1e90ff}")
latex_preamble.append(r"\definecolor{regex_in}{HTML}{ff8c00}")
latex_preamble.append(r"\definecolor{regex_category}{HTML}{8fbc8f}")
latex_preamble.append(r"\definecolor{regex_repeat}{HTML}{FF4500}")
latex_preamble.append(r"\definecolor{regex_any}{HTML}{FF4500}")
latex_elements["preamble"] = str(latex_preamble)
app.config.latex_elements = latex_elements # type: ignore
add_nbsp_substitution(config)
def make_installation_instructions(options: Dict[str, Any], env: BuildEnvironment) -> List[str]:
"""
Make the content of an installation node.
:param options:
:param env: The Sphinx build environment.
"""
tabs: Dict[str, List[str]] = _get_installation_instructions(options, env)
if not tabs:
warnings.warn("No installation source specified. No installation instructions will be shown.")
return []
content = StringList([".. tabs::", ''])
content.set_indent_type(" ")
for tab_name, tab_content in tabs.items():
with content.with_indent_size(1):
content.append(f".. tab:: {tab_name}")
content.blankline(ensure_single=True)
with content.with_indent_size(2):
content.extend([f"{line}" if line else '' for line in tab_content])
return list(content)
def test_prompt(capsys, monkeypatch, data_regression: DataRegressionFixture):
inputs = iter([
'',
'',
'',
'',
"24",
"Bond007",
"badpassword",
"baspassword",
"badpassword",
"badpassword",
"badpassword",
"badpassword",
])
def fake_input(prompt):
value = next(inputs)
print(f"{prompt}{value}".rstrip())
return value
monkeypatch.setattr(click.termui, "visible_prompt_func", fake_input)
assert prompt(text="What is your age", prompt_suffix="? ", type=click.INT) == 24
assert prompt(text="Username", type=click.STRING) == "Bond007"
assert prompt(text="Password", type=click.STRING, confirmation_prompt=True) == "badpassword"
assert prompt(
text="Password",
type=click.STRING,
confirmation_prompt="Are you sure about that? ",
) == "badpassword"
data_regression.check(list(StringList(capsys.readouterr().out.splitlines())))
def document_keys(self, keys: List[str], types: Dict[str, Type],
docstrings: Dict[str, List[str]]):
"""
Document keys in a :class:`typing.TypedDict`.
:param keys: List of key names to document.
:param types: Mapping of key names to types.
:param docstrings: Mapping of key names to docstrings.
"""
content = StringList()
for key in keys:
if key in types:
key_type = f"({format_annotation(types[key])}) "
else:
key_type = ''
if key in docstrings:
content.append(
f" * **{key}** {key_type}-- {' '.join(docstrings.get(key, ''))}"
)
else:
content.append(f" * **{key}** {key_type}")
for line in content:
self.add_line(line, self.get_sourcename())
def test_indent_type(self):
sl = StringList(['', '', "hello", "world", '', '', '', "1234"])
assert sl.indent_type == '\t'
with pytest.raises(ValueError, match="'type' cannot an empty string."):
sl.indent_type = ''
assert sl.indent_type == '\t'
sl.indent_type = ' '
assert sl.indent_type == ' '
sl.set_indent_type('\t')
assert sl.indent_type == '\t'
sl.set_indent_type(' ')
assert sl.indent_type == ' '
with pytest.raises(ValueError, match="'type' cannot an empty string."):
sl.set_indent_type('')
assert sl.indent_type == ' '
sl.set_indent_type()
assert sl.indent_type == '\t'
def __repr__(self) -> str:
buf = StringList()
buf.indent_type = " "
buf.append(f"{self.__class__.__module__}.{self.__class__.__qualname__}(")
with buf.with_indent_size(1):
for attrib in attr.fields(self.__class__):
value = getattr(self, attrib.name)
if isinstance(value, datetime):
buf.append(f"{attrib.name}={value.isoformat()!r},")
elif isinstance(value, str):
lines = textwrap.wrap(value, width=80 - len(attrib.name) - 1)
buf.append(f"{attrib.name}={lines.pop(0)!r}")
for line in lines:
buf.append(' ' * len(attrib.name) + ' ' + repr(line))
buf[-1] = f"{buf[-1][len(buf.indent_type) * buf.indent_size:]},"
elif value is None:
buf.append(f"{attrib.name}=None,")
else:
buf.append(f"{attrib.name}={prettyprinter.pformat(value)},")
buf.append(')')
return str(buf)
def dump_list(self, v) -> str:
"""
Serialize a list to TOML.
:param v:
:rtype:
.. latex:clearpage::
"""
single_line = super().dump_list(v)
if len(single_line) <= self.max_width:
return single_line
retval = StringList(['['])
with retval.with_indent(" ", 1):
for u in v:
retval.append(f"{str(self.dump_value(u))},")
retval.append(']')
return str(retval)
def copy_assets(app: Sphinx, exception: Optional[Exception] = None) -> None:
"""
Copy asset files to the output.
:param app: The Sphinx application.
:param exception: Any exception which occurred and caused Sphinx to abort.
"""
if exception: # pragma: no cover
return
style = StringList([
".docutils.container {",
" padding-left: 0 !important;",
" padding-right: 0 !important;",
'}',
'',
# "div.sphinx-tabs.docutils.container {",
# " padding-left: 0 !important;",
# " padding-right: 0 !important;",
# "}",
# '',
"div.ui.top.attached.tabular.menu.sphinx-menu.docutils.container {",
# " padding-left: 0 !important;",
# " padding-right: 0 !important;",
" margin-left: 0 !important;",
" margin-right: 0 !important;",
'}',
])
css_dir = PathPlus(app.builder.outdir) / "_static" / "css"
css_dir.maybe_make(parents=True)
css_file = css_dir / "tabs_customise.css"
css_file.write_lines(style)
def test_insert(self):
sl = StringList(['', '', "hello", "world", '', '', '', "1234"])
sl.insert(0, "foo")
assert sl == ["foo", '', '', "hello", "world", '', '', '', "1234"]
sl.insert(1, "bar")
assert sl == [
"foo", "bar", '', '', "hello", "world", '', '', '', "1234"
]
sl.insert(0, "1234")
assert sl == [
"1234", "foo", "bar", '', '', "hello", "world", '', '', '', "1234"
]
sl.insert(11, "baz")
assert sl == [
"1234", "foo", "bar", '', '', "hello", "world", '', '', '', "1234",
"baz"
]
sl.insert(3, "\na line\n")
assert sl == [
"1234", "foo", "bar", '', "a line", '', '', '', "hello", "world",
'', '', '', "1234", "baz"
]
sl.insert(100, "end")
assert sl == [
"1234", "foo", "bar", '', "a line", '', '', '', "hello", "world",
'', '', '', "1234", "baz", "end"
]
def test_copy(self):
sl = StringList(['', '', "hello", "world", '', '', '', "1234"])
sl2 = sl.copy()
assert sl == sl2
assert sl2 == ['', '', "hello", "world", '', '', '', "1234"]
assert isinstance(sl2, StringList)
def pformat_tabs(
obj: object,
width: int = 80,
depth: Optional[int] = None,
*,
compact: bool = False,
) -> str:
"""
Format a Python object into a pretty-printed representation.
Indentation is set at one tab.
:param obj: The object to format.
:param width: The maximum width of the output.
:param depth:
:param compact:
"""
prettyprinter = FancyPrinter(indent=4, width=width, depth=depth, compact=compact)
buf = StringList()
for line in prettyprinter.pformat(obj).splitlines():
buf.append(re.sub("^ {4}", r"\t", line))
return str(buf)
def test_blankline(self):
sl = StringList(['', '', "hello", "world", '', '', '', "1234"])
sl.blankline()
assert sl == ['', '', "hello", "world", '', '', '', "1234", '']
sl.blankline()
assert sl == ['', '', "hello", "world", '', '', '', "1234", '', '']
sl.blankline(ensure_single=True)
assert sl == ['', '', "hello", "world", '', '', '', "1234", '']
sl.blankline(ensure_single=True)
assert sl == ['', '', "hello", "world", '', '', '', "1234", '']
sl.append('\t')
sl.blankline(ensure_single=True)
assert sl == ['', '', "hello", "world", '', '', '', "1234", '']
sl.append(" ")
sl.blankline(ensure_single=True)
assert sl == ['', '', "hello", "world", '', '', '', "1234", '']
sl.append(" ")
sl.blankline(ensure_single=True)
sl.blankline()
assert sl == ['', '', "hello", "world", '', '', '', "1234", '', '']
def enable_docs(
repo_path: pathlib.Path,
templates: Environment,
init_repo_templates: Environment,
) -> List[str]:
docs_dir = PathPlus(repo_path / templates.globals["docs_dir"])
docs_dir.maybe_make()
(docs_dir / "api").maybe_make()
for filename in {"index.rst"}:
template = init_repo_templates.get_template(filename)
(docs_dir / filename).write_clean(template.render())
api_buf = StringList()
header_line: str = '=' * (len(templates.globals["import_name"]) + 1)
api_buf.append(header_line)
api_buf.append(templates.globals["import_name"])
api_buf.append(header_line)
api_buf.blankline(ensure_single=True)
api_buf.append(f".. automodule:: {templates.globals['import_name']}")
api_buf.blankline(ensure_single=True)
(docs_dir / "api" /
templates.globals["modname"]).with_suffix(".rst").write_lines(api_buf)
return [
posixpath.join(templates.globals["docs_dir"], "api",
f"{templates.globals['modname']}.rst"),
posixpath.join(templates.globals["docs_dir"], "index.rst"),
]
def format_signature(obj: Union[type, FunctionType]) -> StringList:
"""
Format the signature of the given object, for insertion into the highlight panel.
:param obj:
:return: A list of reStructuredText lines.
"""
with monkeypatcher():
obj.__annotations__ = get_type_hints(obj)
signature: inspect.Signature = inspect.signature(obj)
buf = StringList(".. parsed-literal::")
buf.blankline()
buf.indent_type = " "
buf.indent_size = 1
if signature.return_annotation is not inspect.Signature.empty and not isinstance(
obj, type):
return_annotation = f") -> {format_annotation(signature.return_annotation)}"
else:
return_annotation = f")"
total_length = len(obj.__name__) + len(return_annotation)
arguments_buf: DelimitedList[str] = DelimitedList()
param: inspect.Parameter
for param in signature.parameters.values():
arguments_buf.append(f"{format_parameter(param)}")
total_length += len(arguments_buf[-1])
if total_length <= 60:
signature_buf = StringList(''.join(
[f"{obj.__name__}(", f"{arguments_buf:, }", return_annotation]))
else:
signature_buf = StringList([f"{obj.__name__}("])
signature_buf.indent_type = " "
with signature_buf.with_indent_size(1):
signature_buf.extend(
[f"{arguments_buf:,\n}" + ',', return_annotation])
buf.extend(signature_buf)
return buf
def test_extend(self):
sl = StringList(['', '', "hello", "world", '', '', '', "1234"])
sl.extend(["\nfoo\nbar\n baz"])
assert sl == [
'', '', "hello", "world", '', '', '', "1234", '', "foo", "bar",
" baz"
]
def test_set_indent_error(self):
sl = StringList()
with pytest.raises(
TypeError,
match=
"'size' argument cannot be used when providing an 'Indent' object."
):
sl.set_indent(Indent(0, " "), 5)
def requirement(requirement: str, file: Optional[str] = None) -> int:
"""
Add a requirement.
"""
# 3rd party
from consolekit.utils import abort
from domdf_python_tools.paths import PathPlus, traverse_to_file
from domdf_python_tools.stringlist import StringList
from packaging.requirements import InvalidRequirement
from packaging.specifiers import SpecifierSet
from shippinglabel import normalize_keep_dot
from shippinglabel.requirements import ComparableRequirement, combine_requirements, read_requirements
repo_dir: PathPlus = traverse_to_file(PathPlus.cwd(), "repo_helper.yml",
"git_helper.yml")
if file is None:
requirements_file = repo_dir / "requirements.txt"
if not requirements_file.is_file():
raise abort(f"'{file}' not found.")
else:
requirements_file = PathPlus(file)
if not requirements_file.is_file():
raise abort("'requirements.txt' not found.")
try:
req = ComparableRequirement(requirement)
except InvalidRequirement as e:
raise BadRequirement(requirement, e)
response = (PYPI_API / req.name / "json/").get()
if response.status_code != 200:
raise click.BadParameter(f"No such project {req.name}")
else:
req.name = normalize(response.json()["info"]["name"])
if not req.specifier:
req.specifier = SpecifierSet(
f">={response.json()['info']['version']}")
click.echo(f"Adding requirement '{req}'")
requirements, comments, invalid_lines = read_requirements(
req_file=requirements_file,
include_invalid=True,
normalize_func=normalize_keep_dot,
)
requirements.add(req)
buf = StringList([*comments, *invalid_lines])
buf.extend(str(req) for req in sorted(combine_requirements(requirements)))
requirements_file.write_lines(buf)
return 0
def test_make_style():
style: css_parser.css.CSSStyleRule = make_style(
"li p:last-child", {"max-width": (rem(1200), IMPORTANT)})
assert str(style.selectorText) == "li p:last-child"
assert StringList(style.cssText) == [
"li p:last-child {",
" max-width: 1200rem !important",
" }",
]
serializer = CSSSerializer(trailing_semicolon=True)
with serializer.use():
assert StringList(style.cssText) == [
"li p:last-child {",
"\tmax-width: 1200rem !important;",
'}',
]
def test_latex_output(app, latex_regression: LaTeXRegressionFixture):
assert app.builder.name.lower() == "latex"
with pytest.warns(UserWarning,
match="(No codes specified|No such code 'F401')"):
app.build()
output_file = PathPlus(app.outdir / "python.tex")
latex_regression.check(StringList(output_file.read_lines()), jinja2=True)
def test_creation(self):
assert not StringList()
assert not StringList([])
assert not StringList(())
assert StringList([1]) == ['1']
assert StringList(['1']) == ['1']
assert StringList('1') == ['1']
assert StringList("1\n") == ['1', '']
with pytest.raises(TypeError, match="'int' object is not iterable"):
StringList(1) # type: ignore
def test_negative_getitem(self):
sl = StringList(['', '', "hello", "world", '', '', "abc", "1234"])
assert sl[-1] == "1234"
sl[-1] += "5678"
assert sl == ['', '', "hello", "world", '', '', "abc", "12345678"]
assert sl[-2] == "abc"
sl[-2] += "def"
assert sl == ['', '', "hello", "world", '', '', "abcdef", "12345678"]
def test_isort_stubs(advanced_file_regression: AdvancedFileRegressionFixture):
source = StringList([
'from natsort.natsort import as_ascii as as_ascii, as_utf8 as as_utf8, decoder as decoder, humansorted as '
'humansorted, index_humansorted as index_humansorted, index_natsorted as index_natsorted, index_realsorted as '
'index_realsorted, natsort_key as natsort_key, natsort_keygen as natsort_keygen, natsorted as natsorted, ns as ns, '
'numeric_regex_chooser as numeric_regex_chooser, order_by_index as order_by_index, os_sort_key as os_sort_key, '
'os_sort_keygen as os_sort_keygen, os_sorted as os_sorted, realsorted as realsorted',
"from natsort.utils import chain_functions as chain_functions",
'',
])
advanced_file_regression.check(isort_hook(str(source), "utils.pyi"))
def sort_requirements(filename: PathLike, allow_git: bool = False) -> int:
"""
Sort the requirements in the given file alphabetically.
:param filename: The file to sort the requirements in.
:param allow_git: Whether to allow lines that start with ``git+``, which are allowed by pip but not :pep:`508`.
"""
ret = PASS
filename = PathPlus(filename)
comments: List[str]
requirements: Set[ComparableRequirement]
git_lines: List[str] = []
requirements, comments, invalid_lines = read_requirements(
req_file=filename,
include_invalid=True,
normalize_func=normalize_keep_dot,
)
for line in invalid_lines:
if line.startswith("git+") and allow_git:
git_lines.append(line)
else:
ret |= FAIL
# find and remove pkg-resources==0.0.0
# which is automatically added by broken pip package under Debian
if ComparableRequirement("pkg-resources==0.0.0") in requirements:
requirements.remove(ComparableRequirement("pkg-resources==0.0.0"))
ret |= FAIL
sorted_requirements = sorted(requirements)
buf = StringList(
[*comments, *git_lines, *[str(req) for req in sorted_requirements]])
buf.blankline(ensure_single=True)
if (requirements != sorted_requirements
and buf != filename.read_lines()) or ret:
print('\n'.join(buf))
# print(coloured_diff(
# filename.read_lines(),
# buf,
# str(filename),
# str(filename),
# "(original)",
# "(sorted)",
# lineterm='',
# ))
ret |= FAIL
filename.write_lines(buf)
return ret
