def _render_templates(
self,
package: Package,
file_paths: list[tuple[Path, Path]],
service_name: ServiceName | None = None,
) -> None:
for file_path, template_path in file_paths:
content = render_jinja2_template(
template_path,
package=package,
service_name=service_name,
)
if file_path.suffix in [".py", ".pyi"]:
content = sort_imports(
content,
package.name,
extension="pyi",
third_party=[
"boto3",
"botocore",
"aiobotocore",
*([
package.data.get_service_package_name(i)
for i in package.service_names
] if not service_name else [
package.data.get_service_package_name(service_name)
]),
],
)
content = blackify(content, file_path)
if file_path.suffix == ".md":
content = insert_md_toc(content)
content = fix_pypi_headers(content)
content = format_md(content)
if not file_path.parent.exists():
file_path.parent.mkdir(exist_ok=True, parents=True)
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
self.logger.debug(f"Rendered {NicePath(file_path)}")
def write_service_package(package: ServicePackage, output_path: Path,
generate_setup: bool) -> None:
"""
Create stubs files for service.
Arguments:
package -- Service package.
output_path -- Path to output folder.
generate_setup -- Generate ready-to-install or to-use package.
"""
logger = get_logger()
setup_path = output_path / f"{package.service_name.module_name}_package"
if generate_setup:
package_path = setup_path / package.name
else:
package_path = output_path / package.name
package_path.mkdir(exist_ok=True, parents=True)
templates_path = Path("service")
module_templates_path = templates_path / "service"
file_paths: List[Tuple[Path, Path]] = []
if generate_setup:
file_paths.extend([
(setup_path / "setup.py", templates_path / "setup.py.jinja2"),
(setup_path / "README.md", templates_path / "README.md.jinja2"),
])
file_paths.extend([
(package_path / "version.py",
module_templates_path / "version.py.jinja2"),
(package_path / "__init__.pyi",
module_templates_path / "__init__.pyi.jinja2"),
(package_path / "__init__.py",
module_templates_path / "__init__.pyi.jinja2"),
(package_path / "__main__.py",
module_templates_path / "__main__.py.jinja2"),
(package_path / "py.typed", module_templates_path / "py.typed.jinja2"),
(
package_path / ServiceModuleName.client.stub_file_name,
module_templates_path / ServiceModuleName.client.template_name,
),
(
package_path / ServiceModuleName.client.file_name,
module_templates_path / ServiceModuleName.client.template_name,
),
])
if package.service_resource:
file_paths.extend((
(
package_path /
ServiceModuleName.service_resource.stub_file_name,
module_templates_path /
ServiceModuleName.service_resource.template_name,
),
(
package_path / ServiceModuleName.service_resource.file_name,
module_templates_path /
ServiceModuleName.service_resource.template_name,
),
))
if package.paginators:
file_paths.extend((
(
package_path / ServiceModuleName.paginator.stub_file_name,
module_templates_path /
ServiceModuleName.paginator.template_name,
),
(
package_path / ServiceModuleName.paginator.file_name,
module_templates_path /
ServiceModuleName.paginator.template_name,
),
))
if package.waiters:
file_paths.extend((
(
package_path / ServiceModuleName.waiter.stub_file_name,
module_templates_path / ServiceModuleName.waiter.template_name,
),
(
package_path / ServiceModuleName.waiter.file_name,
module_templates_path / ServiceModuleName.waiter.template_name,
),
))
if package.literals:
file_paths.extend((
(
package_path / ServiceModuleName.literals.stub_file_name,
module_templates_path /
ServiceModuleName.literals.template_name,
),
(
package_path / ServiceModuleName.literals.file_name,
module_templates_path /
ServiceModuleName.literals.template_name,
),
))
if package.typed_dicts:
file_paths.extend((
(
package_path / ServiceModuleName.type_defs.stub_file_name,
module_templates_path /
ServiceModuleName.type_defs.template_name,
),
(
package_path / ServiceModuleName.type_defs.file_name,
module_templates_path /
ServiceModuleName.type_defs.template_name,
),
))
for file_path, template_path in file_paths:
content = render_jinja2_template(
template_path,
package=package,
service_name=package.service_name,
)
if file_path.suffix in [".py", ".pyi"]:
content = sort_imports(content,
package.service_name.module_name,
extension="pyi")
content = blackify(content, file_path)
if file_path.suffix == ".md":
content = insert_md_toc(content)
content = fix_pypi_headers(content)
content = format_md(content)
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
logger.debug(f"Updated {NicePath(file_path)}")
valid_paths = dict(file_paths).keys()
for unknown_path in NicePath(
setup_path if generate_setup else package_path).walk(valid_paths):
unknown_path.unlink()
logger.debug(f"Deleted {NicePath(unknown_path)}")
def write_boto3_stubs_package(package: Boto3StubsPackage, output_path: Path,
generate_setup: bool) -> List[Path]:
setup_path = output_path / "boto3_stubs_package"
if not generate_setup:
setup_path = output_path
modified_paths: List[Path] = []
package_path = setup_path / package.name
if not generate_setup:
package_path = setup_path / "boto3"
if setup_path.exists():
shutil.rmtree(setup_path)
setup_path.mkdir(exist_ok=True)
package_path.mkdir(exist_ok=True)
templates_path = Path("boto3-stubs")
module_templates_path = templates_path / "boto3-stubs"
file_paths: List[Tuple[Path, Path]] = []
if generate_setup:
file_paths.extend([
(setup_path / "setup.py", templates_path / "setup.py.jinja2"),
(setup_path / "README.md", templates_path / "README.md.jinja2"),
])
file_paths.extend([
(package_path / "py.typed", module_templates_path / "py.typed.jinja2"),
(package_path / "__init__.pyi",
module_templates_path / "__init__.pyi.jinja2"),
(package_path / "__init__.py",
module_templates_path / "__init__.pyi.jinja2"),
(package_path / "session.pyi",
module_templates_path / "session.pyi.jinja2"),
(package_path / "__main__.py",
module_templates_path / "__main__.py.jinja2"),
(package_path / "version.py",
module_templates_path / "version.py.jinja2"),
])
for file_path, template_path in file_paths:
content = render_jinja2_template(template_path, package=package)
content = blackify(content, file_path)
content = sort_imports(
content,
"boto3_stubs",
extension="pyi",
third_party=[
"boto3", "botocore",
*[i.module_name for i in package.service_names]
],
)
if not file_path.exists() or file_path.read_text() != content:
modified_paths.append(file_path)
file_path.write_text(content)
for static_path in BOTO3_STUBS_STATIC_PATH.glob("**/*.pyi"):
relative_output_path = static_path.relative_to(BOTO3_STUBS_STATIC_PATH)
file_path = package_path / relative_output_path
file_path.parent.mkdir(exist_ok=True)
if file_path.exists() and filecmp.cmp(static_path.as_posix(),
file_path.as_posix()):
continue
shutil.copy(static_path, file_path)
modified_paths.append(file_path)
return modified_paths
def write_master_package(package: MasterPackage, output_path: Path,
generate_setup: bool) -> List[Path]:
setup_path = output_path / "master_package"
if not generate_setup:
setup_path = output_path
modified_paths: List[Path] = []
package_path = setup_path / package.name
if setup_path.exists():
shutil.rmtree(setup_path)
setup_path.mkdir(exist_ok=True)
package_path.mkdir(exist_ok=True)
templates_path = Path("master")
module_templates_path = templates_path / "master"
file_paths: List[Tuple[Path, Path]] = []
if generate_setup:
file_paths.extend([
(setup_path / "setup.py", templates_path / "setup.py.jinja2"),
(setup_path / "README.md", templates_path / "README.md.jinja2"),
])
file_paths.extend([
(package_path / "__init__.py",
module_templates_path / "__init__.py.jinja2"),
(package_path / "__main__.py",
module_templates_path / "__main__.py.jinja2"),
(package_path / "py.typed", module_templates_path / "py.typed.jinja2"),
(package_path / "version.py",
module_templates_path / "version.py.jinja2"),
(package_path / "main.py", module_templates_path / "main.py.jinja2"),
(package_path / "boto3_init.py",
module_templates_path / "boto3_init.py.jinja2"),
(package_path / "boto3_session.py",
module_templates_path / "boto3_session.py.jinja2"),
(
package_path / "boto3_init_gen.py",
module_templates_path / "boto3_init_stub.py.jinja2",
),
(
package_path / "boto3_session_gen.py",
module_templates_path / "boto3_session_stub.py.jinja2",
),
(
package_path / "boto3_init_stub.py",
module_templates_path / "boto3_init_stub.py.jinja2",
),
(
package_path / "boto3_session_stub.py",
module_templates_path / "boto3_session_stub.py.jinja2",
),
(package_path / "submodules.py",
module_templates_path / "submodules.py.jinja2"),
])
for file_path, template_path in file_paths:
content = render_jinja2_template(template_path, package=package)
content = blackify(content, file_path)
content = sort_imports(content,
"mypy_boto3",
extension=file_path.suffix[1:])
if not file_path.exists() or file_path.read_text() != content:
modified_paths.append(file_path)
file_path.write_text(content)
return modified_paths
def test_sort_imports(self, sort_code_string_mock: MagicMock):
sort_code_string_mock.return_value = "output"
assert sort_imports("test", "mymodule") == "output"
sort_code_string_mock.assert_called()
assert sort_imports("test", "boto3") == "output"
def write_service_package(
package: ServicePackage, output_path: Path, generate_setup: bool
) -> List[Path]:
setup_path = output_path / f"{package.service_name.module_name}_package"
if not generate_setup:
setup_path = output_path
modified_paths: List[Path] = []
package_path = setup_path / package.name
if setup_path.exists():
shutil.rmtree(setup_path)
setup_path.mkdir(exist_ok=True)
package_path.mkdir(exist_ok=True)
templates_path = Path("service")
module_templates_path = templates_path / "service"
file_paths: List[Tuple[Path, Path]] = []
if generate_setup:
file_paths.extend(
[
(setup_path / "setup.py", templates_path / "setup.py.jinja2"),
(setup_path / "README.md", templates_path / "README.md.jinja2"),
]
)
file_paths.extend(
[
(package_path / "version.py", module_templates_path / "version.py.jinja2"),
(package_path / "__init__.pyi", module_templates_path / "__init__.pyi.jinja2"),
(package_path / "__init__.py", module_templates_path / "__init__.pyi.jinja2"),
(package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"),
(package_path / "py.typed", module_templates_path / "py.typed.jinja2"),
(
package_path / ServiceModuleName.client.stub_file_name,
module_templates_path / ServiceModuleName.client.template_name,
),
(
package_path / ServiceModuleName.client.file_name,
module_templates_path / ServiceModuleName.client.template_name,
),
]
)
if package.service_resource:
file_paths.extend(
(
(
package_path / ServiceModuleName.service_resource.stub_file_name,
module_templates_path / ServiceModuleName.service_resource.template_name,
),
(
package_path / ServiceModuleName.service_resource.file_name,
module_templates_path / ServiceModuleName.service_resource.template_name,
),
)
)
if package.paginators:
file_paths.extend(
(
(
package_path / ServiceModuleName.paginator.stub_file_name,
module_templates_path / ServiceModuleName.paginator.template_name,
),
(
package_path / ServiceModuleName.paginator.file_name,
module_templates_path / ServiceModuleName.paginator.template_name,
),
)
)
if package.waiters:
file_paths.extend(
(
(
package_path / ServiceModuleName.waiter.stub_file_name,
module_templates_path / ServiceModuleName.waiter.template_name,
),
(
package_path / ServiceModuleName.waiter.file_name,
module_templates_path / ServiceModuleName.waiter.template_name,
),
)
)
if package.typed_dicts:
file_paths.extend(
(
(
package_path / ServiceModuleName.type_defs.stub_file_name,
module_templates_path / ServiceModuleName.type_defs.template_name,
),
(
package_path / ServiceModuleName.type_defs.file_name,
module_templates_path / ServiceModuleName.type_defs.template_name,
),
)
)
for file_path, template_path in file_paths:
content = render_jinja2_template(
template_path,
package=package,
service_name=package.service_name,
)
content = blackify(content, file_path)
content = sort_imports(content, package.service_name.module_name, extension="pyi")
if not file_path.exists() or file_path.read_text() != content:
modified_paths.append(file_path)
file_path.write_text(content)
return modified_paths
def write_botocore_stubs_package(output_path: Path,
generate_setup: bool) -> None:
"""
Generate botocore-stubs stub files.
Arguments:
output_path -- Path to output folder.
generate_setup -- Generate ready-to-install or to-use package.
"""
logger = get_logger()
setup_path = output_path / "botocore_stubs_package"
if generate_setup:
package_path = setup_path / BOTOCORE_STUBS_NAME
else:
package_path = output_path / "botocore"
package_path.mkdir(exist_ok=True, parents=True)
templates_path = Path("botocore-stubs")
module_templates_path = templates_path / "botocore-stubs"
file_paths: List[Tuple[Path, Path]] = []
if generate_setup:
file_paths.extend([
(setup_path / "setup.py", templates_path / "setup.py.jinja2"),
(setup_path / "README.md", templates_path / "README.md.jinja2"),
])
file_paths.extend([
(package_path / "py.typed", module_templates_path / "py.typed.jinja2"),
(package_path / "__main__.py",
module_templates_path / "__main__.py.jinja2"),
(package_path / "version.py",
module_templates_path / "version.py.jinja2"),
])
for file_path, template_path in file_paths:
content = render_jinja2_template(template_path)
if file_path.suffix in [".py", ".pyi"]:
content = sort_imports(
content,
"boto3_stubs",
extension="pyi",
)
content = blackify(content, file_path)
if file_path.suffix == ".md":
content = insert_md_toc(content)
content = fix_pypi_headers(content)
content = format_md(content)
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
logger.debug(f"Updated {NicePath(file_path)}")
static_paths = []
for static_path in BOTOCORE_STUBS_STATIC_PATH.glob("**/*.pyi"):
relative_output_path = static_path.relative_to(
BOTOCORE_STUBS_STATIC_PATH)
file_path = package_path / relative_output_path
static_paths.append(file_path)
file_path.parent.mkdir(exist_ok=True)
content = static_path.read_text()
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
logger.debug(f"Updated {NicePath(file_path)}")
valid_paths = (*dict(file_paths).keys(), *static_paths)
for unknown_path in NicePath(
setup_path if generate_setup else package_path).walk(valid_paths):
unknown_path.unlink()
logger.debug(f"Deleted {NicePath(unknown_path)}")
def write_master_package(package: MasterPackage, output_path: Path, generate_setup: bool) -> None:
"""
Create mypy-boto3 stubs.
Arguments:
package -- Master package.
output_path -- Path to output folder.
generate_setup -- Generate ready-to-install or to-use package.
"""
logger = get_logger()
setup_path = output_path / "master_package"
if generate_setup:
package_path = setup_path / package.name
else:
package_path = output_path / package.name
package_path.mkdir(exist_ok=True, parents=True)
templates_path = Path("master")
module_templates_path = templates_path / "master"
file_paths: List[Tuple[Path, Path]] = []
if generate_setup:
file_paths.extend(
[
(setup_path / "setup.py", templates_path / "setup.py.jinja2"),
(setup_path / "README.md", templates_path / "README.md.jinja2"),
]
)
file_paths.extend(
[
(package_path / "__init__.py", module_templates_path / "__init__.py.jinja2"),
(package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"),
(package_path / "py.typed", module_templates_path / "py.typed.jinja2"),
(package_path / "version.py", module_templates_path / "version.py.jinja2"),
(package_path / "main.py", module_templates_path / "main.py.jinja2"),
(package_path / "boto3_init.py", module_templates_path / "boto3_init.py.jinja2"),
(package_path / "boto3_session.py", module_templates_path / "boto3_session.py.jinja2"),
(
package_path / "boto3_init_gen.py",
module_templates_path / "boto3_init_stub.py.jinja2",
),
(
package_path / "boto3_session_gen.py",
module_templates_path / "boto3_session_stub.py.jinja2",
),
(
package_path / "boto3_init_stub.py",
module_templates_path / "boto3_init_stub.py.jinja2",
),
(
package_path / "boto3_session_stub.py",
module_templates_path / "boto3_session_stub.py.jinja2",
),
(package_path / "submodules.py", module_templates_path / "submodules.py.jinja2"),
]
)
for file_path, template_path in file_paths:
content = render_jinja2_template(template_path, package=package)
if file_path.suffix in [".py", ".pyi"]:
content = sort_imports(content, "mypy_boto3", extension=file_path.suffix[1:])
content = blackify(content, file_path)
if file_path.suffix == ".md":
content = insert_md_toc(content)
content = fix_pypi_headers(content)
content = format_md(content)
if not file_path.exists() or file_path.read_text() != content:
file_path.write_text(content)
logger.debug(f"Updated {NicePath(file_path)}")
valid_paths = dict(file_paths).keys()
for unknown_path in NicePath(setup_path if generate_setup else package_path).walk(valid_paths):
unknown_path.unlink()
logger.debug(f"Deleted {NicePath(unknown_path)}")