def write_service_docs(package: ServicePackage, output_path: Path) -> None:
"""
Create service docs files.
Arguments:
package -- Service package.
output_path -- Path to output folder.
"""
logger = get_logger()
docs_path = output_path / f"{package.service_name.module_name}"
docs_path.mkdir(exist_ok=True)
templates_path = Path("service_docs")
file_paths = [
(docs_path / "README.md", templates_path / "README.md.jinja2"),
(docs_path / "client.md", templates_path / "client.md.jinja2"),
]
if package.literals:
file_paths.append(
(docs_path / "literals.md", templates_path / "literals.md.jinja2"))
if package.typed_dicts:
file_paths.append((docs_path / "type_defs.md",
templates_path / "type_defs.md.jinja2"))
if package.waiters:
file_paths.append(
(docs_path / "waiters.md", templates_path / "waiters.md.jinja2"))
if package.paginators:
file_paths.append((docs_path / "paginators.md",
templates_path / "paginators.md.jinja2"))
if package.service_resource:
file_paths.append((docs_path / "service_resource.md",
templates_path / "service_resource.md.jinja2"))
for file_path, template_path in file_paths:
content = render_jinja2_template(
template_path,
package=package,
service_name=package.service_name,
)
content = insert_md_toc(content)
content = format_md(content)
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
logger.debug(f"Updated {NicePath(file_path)}")
valid_paths = dict(file_paths).keys()
for unknown_path in NicePath(docs_path).walk(valid_paths):
unknown_path.unlink()
logger.debug(f"Deleted {NicePath(unknown_path)}")
def write_docs(self, package: Package, templates_path: Path) -> None:
"""
Generate docs for a package.
"""
self.output_path.mkdir(exist_ok=True, parents=True)
file_paths = []
for template_path in templates_path.glob("**/*.jinja2"):
file_name = template_path.name.rsplit(".", 1)[0]
file_paths.append((self.output_path / file_name, template_path))
for file_path, template_path in file_paths:
content = render_jinja2_template(
template_path,
package=package,
)
content = insert_md_toc(content)
content = format_md(content)
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
self.logger.debug(f"Updated {NicePath(file_path)}")
def _render_templates(
self,
package: Package,
file_paths: list[tuple[Path, Path]],
service_name: ServiceName | None = None,
) -> None:
for file_path, template_path in file_paths:
content = render_jinja2_template(
template_path,
package=package,
service_name=service_name,
)
if file_path.suffix in [".py", ".pyi"]:
content = sort_imports(
content,
package.name,
extension="pyi",
third_party=[
"boto3",
"botocore",
"aiobotocore",
*([
package.data.get_service_package_name(i)
for i in package.service_names
] if not service_name else [
package.data.get_service_package_name(service_name)
]),
],
)
content = blackify(content, file_path)
if file_path.suffix == ".md":
content = insert_md_toc(content)
content = fix_pypi_headers(content)
content = format_md(content)
if not file_path.parent.exists():
file_path.parent.mkdir(exist_ok=True, parents=True)
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
self.logger.debug(f"Rendered {NicePath(file_path)}")
def write_service_package(package: ServicePackage, output_path: Path,
generate_setup: bool) -> None:
"""
Create stubs files for service.
Arguments:
package -- Service package.
output_path -- Path to output folder.
generate_setup -- Generate ready-to-install or to-use package.
"""
logger = get_logger()
setup_path = output_path / f"{package.service_name.module_name}_package"
if generate_setup:
package_path = setup_path / package.name
else:
package_path = output_path / package.name
package_path.mkdir(exist_ok=True, parents=True)
templates_path = Path("service")
module_templates_path = templates_path / "service"
file_paths: List[Tuple[Path, Path]] = []
if generate_setup:
file_paths.extend([
(setup_path / "setup.py", templates_path / "setup.py.jinja2"),
(setup_path / "README.md", templates_path / "README.md.jinja2"),
])
file_paths.extend([
(package_path / "version.py",
module_templates_path / "version.py.jinja2"),
(package_path / "__init__.pyi",
module_templates_path / "__init__.pyi.jinja2"),
(package_path / "__init__.py",
module_templates_path / "__init__.pyi.jinja2"),
(package_path / "__main__.py",
module_templates_path / "__main__.py.jinja2"),
(package_path / "py.typed", module_templates_path / "py.typed.jinja2"),
(
package_path / ServiceModuleName.client.stub_file_name,
module_templates_path / ServiceModuleName.client.template_name,
),
(
package_path / ServiceModuleName.client.file_name,
module_templates_path / ServiceModuleName.client.template_name,
),
])
if package.service_resource:
file_paths.extend((
(
package_path /
ServiceModuleName.service_resource.stub_file_name,
module_templates_path /
ServiceModuleName.service_resource.template_name,
),
(
package_path / ServiceModuleName.service_resource.file_name,
module_templates_path /
ServiceModuleName.service_resource.template_name,
),
))
if package.paginators:
file_paths.extend((
(
package_path / ServiceModuleName.paginator.stub_file_name,
module_templates_path /
ServiceModuleName.paginator.template_name,
),
(
package_path / ServiceModuleName.paginator.file_name,
module_templates_path /
ServiceModuleName.paginator.template_name,
),
))
if package.waiters:
file_paths.extend((
(
package_path / ServiceModuleName.waiter.stub_file_name,
module_templates_path / ServiceModuleName.waiter.template_name,
),
(
package_path / ServiceModuleName.waiter.file_name,
module_templates_path / ServiceModuleName.waiter.template_name,
),
))
if package.literals:
file_paths.extend((
(
package_path / ServiceModuleName.literals.stub_file_name,
module_templates_path /
ServiceModuleName.literals.template_name,
),
(
package_path / ServiceModuleName.literals.file_name,
module_templates_path /
ServiceModuleName.literals.template_name,
),
))
if package.typed_dicts:
file_paths.extend((
(
package_path / ServiceModuleName.type_defs.stub_file_name,
module_templates_path /
ServiceModuleName.type_defs.template_name,
),
(
package_path / ServiceModuleName.type_defs.file_name,
module_templates_path /
ServiceModuleName.type_defs.template_name,
),
))
for file_path, template_path in file_paths:
content = render_jinja2_template(
template_path,
package=package,
service_name=package.service_name,
)
if file_path.suffix in [".py", ".pyi"]:
content = sort_imports(content,
package.service_name.module_name,
extension="pyi")
content = blackify(content, file_path)
if file_path.suffix == ".md":
content = insert_md_toc(content)
content = fix_pypi_headers(content)
content = format_md(content)
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
logger.debug(f"Updated {NicePath(file_path)}")
valid_paths = dict(file_paths).keys()
for unknown_path in NicePath(
setup_path if generate_setup else package_path).walk(valid_paths):
unknown_path.unlink()
logger.debug(f"Deleted {NicePath(unknown_path)}")
def test_insert_md_toc(self) -> None:
assert (
insert_md_toc("# a\ntest\n## b\n## c\ntest2") ==
"# a\ntest\n- [a](#a)\n - [b](#b)\n - [c](#c)\n\n## b\n## c\ntest2"
)
assert insert_md_toc("# a\n") == "# a\n- [a](#a)\n"
def write_botocore_stubs_package(output_path: Path,
generate_setup: bool) -> None:
"""
Generate botocore-stubs stub files.
Arguments:
output_path -- Path to output folder.
generate_setup -- Generate ready-to-install or to-use package.
"""
logger = get_logger()
setup_path = output_path / "botocore_stubs_package"
if generate_setup:
package_path = setup_path / BOTOCORE_STUBS_NAME
else:
package_path = output_path / "botocore"
package_path.mkdir(exist_ok=True, parents=True)
templates_path = Path("botocore-stubs")
module_templates_path = templates_path / "botocore-stubs"
file_paths: List[Tuple[Path, Path]] = []
if generate_setup:
file_paths.extend([
(setup_path / "setup.py", templates_path / "setup.py.jinja2"),
(setup_path / "README.md", templates_path / "README.md.jinja2"),
])
file_paths.extend([
(package_path / "py.typed", module_templates_path / "py.typed.jinja2"),
(package_path / "__main__.py",
module_templates_path / "__main__.py.jinja2"),
(package_path / "version.py",
module_templates_path / "version.py.jinja2"),
])
for file_path, template_path in file_paths:
content = render_jinja2_template(template_path)
if file_path.suffix in [".py", ".pyi"]:
content = sort_imports(
content,
"boto3_stubs",
extension="pyi",
)
content = blackify(content, file_path)
if file_path.suffix == ".md":
content = insert_md_toc(content)
content = fix_pypi_headers(content)
content = format_md(content)
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
logger.debug(f"Updated {NicePath(file_path)}")
static_paths = []
for static_path in BOTOCORE_STUBS_STATIC_PATH.glob("**/*.pyi"):
relative_output_path = static_path.relative_to(
BOTOCORE_STUBS_STATIC_PATH)
file_path = package_path / relative_output_path
static_paths.append(file_path)
file_path.parent.mkdir(exist_ok=True)
content = static_path.read_text()
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
logger.debug(f"Updated {NicePath(file_path)}")
valid_paths = (*dict(file_paths).keys(), *static_paths)
for unknown_path in NicePath(
setup_path if generate_setup else package_path).walk(valid_paths):
unknown_path.unlink()
logger.debug(f"Deleted {NicePath(unknown_path)}")
def write_master_package(package: MasterPackage, output_path: Path, generate_setup: bool) -> None:
"""
Create mypy-boto3 stubs.
Arguments:
package -- Master package.
output_path -- Path to output folder.
generate_setup -- Generate ready-to-install or to-use package.
"""
logger = get_logger()
setup_path = output_path / "master_package"
if generate_setup:
package_path = setup_path / package.name
else:
package_path = output_path / package.name
package_path.mkdir(exist_ok=True, parents=True)
templates_path = Path("master")
module_templates_path = templates_path / "master"
file_paths: List[Tuple[Path, Path]] = []
if generate_setup:
file_paths.extend(
[
(setup_path / "setup.py", templates_path / "setup.py.jinja2"),
(setup_path / "README.md", templates_path / "README.md.jinja2"),
]
)
file_paths.extend(
[
(package_path / "__init__.py", module_templates_path / "__init__.py.jinja2"),
(package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"),
(package_path / "py.typed", module_templates_path / "py.typed.jinja2"),
(package_path / "version.py", module_templates_path / "version.py.jinja2"),
(package_path / "main.py", module_templates_path / "main.py.jinja2"),
(package_path / "boto3_init.py", module_templates_path / "boto3_init.py.jinja2"),
(package_path / "boto3_session.py", module_templates_path / "boto3_session.py.jinja2"),
(
package_path / "boto3_init_gen.py",
module_templates_path / "boto3_init_stub.py.jinja2",
),
(
package_path / "boto3_session_gen.py",
module_templates_path / "boto3_session_stub.py.jinja2",
),
(
package_path / "boto3_init_stub.py",
module_templates_path / "boto3_init_stub.py.jinja2",
),
(
package_path / "boto3_session_stub.py",
module_templates_path / "boto3_session_stub.py.jinja2",
),
(package_path / "submodules.py", module_templates_path / "submodules.py.jinja2"),
]
)
for file_path, template_path in file_paths:
content = render_jinja2_template(template_path, package=package)
if file_path.suffix in [".py", ".pyi"]:
content = sort_imports(content, "mypy_boto3", extension=file_path.suffix[1:])
content = blackify(content, file_path)
if file_path.suffix == ".md":
content = insert_md_toc(content)
content = fix_pypi_headers(content)
content = format_md(content)
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
logger.debug(f"Updated {NicePath(file_path)}")
valid_paths = dict(file_paths).keys()
for unknown_path in NicePath(setup_path if generate_setup else package_path).walk(valid_paths):
unknown_path.unlink()
logger.debug(f"Deleted {NicePath(unknown_path)}")