def _render_templates( self, package: Package, file_paths: list[tuple[Path, Path]], service_name: ServiceName | None = None, ) -> None: for file_path, template_path in file_paths: content = render_jinja2_template( template_path, package=package, service_name=service_name, ) if file_path.suffix in [".py", ".pyi"]: content = sort_imports( content, package.name, extension="pyi", third_party=[ "boto3", "botocore", "aiobotocore", *([ package.data.get_service_package_name(i) for i in package.service_names ] if not service_name else [ package.data.get_service_package_name(service_name) ]), ], ) content = blackify(content, file_path) if file_path.suffix == ".md": content = insert_md_toc(content) content = fix_pypi_headers(content) content = format_md(content) if not file_path.parent.exists(): file_path.parent.mkdir(exist_ok=True, parents=True) if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) self.logger.debug(f"Rendered {NicePath(file_path)}")
def write_service_package(package: ServicePackage, output_path: Path, generate_setup: bool) -> None: """ Create stubs files for service. Arguments: package -- Service package. output_path -- Path to output folder. generate_setup -- Generate ready-to-install or to-use package. """ logger = get_logger() setup_path = output_path / f"{package.service_name.module_name}_package" if generate_setup: package_path = setup_path / package.name else: package_path = output_path / package.name package_path.mkdir(exist_ok=True, parents=True) templates_path = Path("service") module_templates_path = templates_path / "service" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend([ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ]) file_paths.extend([ (package_path / "version.py", module_templates_path / "version.py.jinja2"), (package_path / "__init__.pyi", module_templates_path / "__init__.pyi.jinja2"), (package_path / "__init__.py", module_templates_path / "__init__.pyi.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), ( package_path / ServiceModuleName.client.stub_file_name, module_templates_path / ServiceModuleName.client.template_name, ), ( package_path / ServiceModuleName.client.file_name, module_templates_path / ServiceModuleName.client.template_name, ), ]) if package.service_resource: file_paths.extend(( ( package_path / ServiceModuleName.service_resource.stub_file_name, module_templates_path / ServiceModuleName.service_resource.template_name, ), ( package_path / ServiceModuleName.service_resource.file_name, module_templates_path / ServiceModuleName.service_resource.template_name, ), )) if package.paginators: file_paths.extend(( ( package_path / ServiceModuleName.paginator.stub_file_name, module_templates_path / ServiceModuleName.paginator.template_name, ), ( package_path / ServiceModuleName.paginator.file_name, module_templates_path / ServiceModuleName.paginator.template_name, ), )) if package.waiters: file_paths.extend(( ( package_path / ServiceModuleName.waiter.stub_file_name, module_templates_path / ServiceModuleName.waiter.template_name, ), ( package_path / ServiceModuleName.waiter.file_name, module_templates_path / ServiceModuleName.waiter.template_name, ), )) if package.literals: file_paths.extend(( ( package_path / ServiceModuleName.literals.stub_file_name, module_templates_path / ServiceModuleName.literals.template_name, ), ( package_path / ServiceModuleName.literals.file_name, module_templates_path / ServiceModuleName.literals.template_name, ), )) if package.typed_dicts: file_paths.extend(( ( package_path / ServiceModuleName.type_defs.stub_file_name, module_templates_path / ServiceModuleName.type_defs.template_name, ), ( package_path / ServiceModuleName.type_defs.file_name, module_templates_path / ServiceModuleName.type_defs.template_name, ), )) for file_path, template_path in file_paths: content = render_jinja2_template( template_path, package=package, service_name=package.service_name, ) if file_path.suffix in [".py", ".pyi"]: content = sort_imports(content, package.service_name.module_name, extension="pyi") content = blackify(content, file_path) if file_path.suffix == ".md": content = insert_md_toc(content) content = fix_pypi_headers(content) content = format_md(content) if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) logger.debug(f"Updated {NicePath(file_path)}") valid_paths = dict(file_paths).keys() for unknown_path in NicePath( setup_path if generate_setup else package_path).walk(valid_paths): unknown_path.unlink() logger.debug(f"Deleted {NicePath(unknown_path)}")
def write_boto3_stubs_package(package: Boto3StubsPackage, output_path: Path, generate_setup: bool) -> List[Path]: setup_path = output_path / "boto3_stubs_package" if not generate_setup: setup_path = output_path modified_paths: List[Path] = [] package_path = setup_path / package.name if not generate_setup: package_path = setup_path / "boto3" if setup_path.exists(): shutil.rmtree(setup_path) setup_path.mkdir(exist_ok=True) package_path.mkdir(exist_ok=True) templates_path = Path("boto3-stubs") module_templates_path = templates_path / "boto3-stubs" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend([ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ]) file_paths.extend([ (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), (package_path / "__init__.pyi", module_templates_path / "__init__.pyi.jinja2"), (package_path / "__init__.py", module_templates_path / "__init__.pyi.jinja2"), (package_path / "session.pyi", module_templates_path / "session.pyi.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), ]) for file_path, template_path in file_paths: content = render_jinja2_template(template_path, package=package) content = blackify(content, file_path) content = sort_imports( content, "boto3_stubs", extension="pyi", third_party=[ "boto3", "botocore", *[i.module_name for i in package.service_names] ], ) if not file_path.exists() or file_path.read_text() != content: modified_paths.append(file_path) file_path.write_text(content) for static_path in BOTO3_STUBS_STATIC_PATH.glob("**/*.pyi"): relative_output_path = static_path.relative_to(BOTO3_STUBS_STATIC_PATH) file_path = package_path / relative_output_path file_path.parent.mkdir(exist_ok=True) if file_path.exists() and filecmp.cmp(static_path.as_posix(), file_path.as_posix()): continue shutil.copy(static_path, file_path) modified_paths.append(file_path) return modified_paths
def write_master_package(package: MasterPackage, output_path: Path, generate_setup: bool) -> List[Path]: setup_path = output_path / "master_package" if not generate_setup: setup_path = output_path modified_paths: List[Path] = [] package_path = setup_path / package.name if setup_path.exists(): shutil.rmtree(setup_path) setup_path.mkdir(exist_ok=True) package_path.mkdir(exist_ok=True) templates_path = Path("master") module_templates_path = templates_path / "master" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend([ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ]) file_paths.extend([ (package_path / "__init__.py", module_templates_path / "__init__.py.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), (package_path / "main.py", module_templates_path / "main.py.jinja2"), (package_path / "boto3_init.py", module_templates_path / "boto3_init.py.jinja2"), (package_path / "boto3_session.py", module_templates_path / "boto3_session.py.jinja2"), ( package_path / "boto3_init_gen.py", module_templates_path / "boto3_init_stub.py.jinja2", ), ( package_path / "boto3_session_gen.py", module_templates_path / "boto3_session_stub.py.jinja2", ), ( package_path / "boto3_init_stub.py", module_templates_path / "boto3_init_stub.py.jinja2", ), ( package_path / "boto3_session_stub.py", module_templates_path / "boto3_session_stub.py.jinja2", ), (package_path / "submodules.py", module_templates_path / "submodules.py.jinja2"), ]) for file_path, template_path in file_paths: content = render_jinja2_template(template_path, package=package) content = blackify(content, file_path) content = sort_imports(content, "mypy_boto3", extension=file_path.suffix[1:]) if not file_path.exists() or file_path.read_text() != content: modified_paths.append(file_path) file_path.write_text(content) return modified_paths
def test_sort_imports(self, sort_code_string_mock: MagicMock): sort_code_string_mock.return_value = "output" assert sort_imports("test", "mymodule") == "output" sort_code_string_mock.assert_called() assert sort_imports("test", "boto3") == "output"
def write_service_package( package: ServicePackage, output_path: Path, generate_setup: bool ) -> List[Path]: setup_path = output_path / f"{package.service_name.module_name}_package" if not generate_setup: setup_path = output_path modified_paths: List[Path] = [] package_path = setup_path / package.name if setup_path.exists(): shutil.rmtree(setup_path) setup_path.mkdir(exist_ok=True) package_path.mkdir(exist_ok=True) templates_path = Path("service") module_templates_path = templates_path / "service" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend( [ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ] ) file_paths.extend( [ (package_path / "version.py", module_templates_path / "version.py.jinja2"), (package_path / "__init__.pyi", module_templates_path / "__init__.pyi.jinja2"), (package_path / "__init__.py", module_templates_path / "__init__.pyi.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), ( package_path / ServiceModuleName.client.stub_file_name, module_templates_path / ServiceModuleName.client.template_name, ), ( package_path / ServiceModuleName.client.file_name, module_templates_path / ServiceModuleName.client.template_name, ), ] ) if package.service_resource: file_paths.extend( ( ( package_path / ServiceModuleName.service_resource.stub_file_name, module_templates_path / ServiceModuleName.service_resource.template_name, ), ( package_path / ServiceModuleName.service_resource.file_name, module_templates_path / ServiceModuleName.service_resource.template_name, ), ) ) if package.paginators: file_paths.extend( ( ( package_path / ServiceModuleName.paginator.stub_file_name, module_templates_path / ServiceModuleName.paginator.template_name, ), ( package_path / ServiceModuleName.paginator.file_name, module_templates_path / ServiceModuleName.paginator.template_name, ), ) ) if package.waiters: file_paths.extend( ( ( package_path / ServiceModuleName.waiter.stub_file_name, module_templates_path / ServiceModuleName.waiter.template_name, ), ( package_path / ServiceModuleName.waiter.file_name, module_templates_path / ServiceModuleName.waiter.template_name, ), ) ) if package.typed_dicts: file_paths.extend( ( ( package_path / ServiceModuleName.type_defs.stub_file_name, module_templates_path / ServiceModuleName.type_defs.template_name, ), ( package_path / ServiceModuleName.type_defs.file_name, module_templates_path / ServiceModuleName.type_defs.template_name, ), ) ) for file_path, template_path in file_paths: content = render_jinja2_template( template_path, package=package, service_name=package.service_name, ) content = blackify(content, file_path) content = sort_imports(content, package.service_name.module_name, extension="pyi") if not file_path.exists() or file_path.read_text() != content: modified_paths.append(file_path) file_path.write_text(content) return modified_paths
def write_botocore_stubs_package(output_path: Path, generate_setup: bool) -> None: """ Generate botocore-stubs stub files. Arguments: output_path -- Path to output folder. generate_setup -- Generate ready-to-install or to-use package. """ logger = get_logger() setup_path = output_path / "botocore_stubs_package" if generate_setup: package_path = setup_path / BOTOCORE_STUBS_NAME else: package_path = output_path / "botocore" package_path.mkdir(exist_ok=True, parents=True) templates_path = Path("botocore-stubs") module_templates_path = templates_path / "botocore-stubs" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend([ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ]) file_paths.extend([ (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), ]) for file_path, template_path in file_paths: content = render_jinja2_template(template_path) if file_path.suffix in [".py", ".pyi"]: content = sort_imports( content, "boto3_stubs", extension="pyi", ) content = blackify(content, file_path) if file_path.suffix == ".md": content = insert_md_toc(content) content = fix_pypi_headers(content) content = format_md(content) if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) logger.debug(f"Updated {NicePath(file_path)}") static_paths = [] for static_path in BOTOCORE_STUBS_STATIC_PATH.glob("**/*.pyi"): relative_output_path = static_path.relative_to( BOTOCORE_STUBS_STATIC_PATH) file_path = package_path / relative_output_path static_paths.append(file_path) file_path.parent.mkdir(exist_ok=True) content = static_path.read_text() if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) logger.debug(f"Updated {NicePath(file_path)}") valid_paths = (*dict(file_paths).keys(), *static_paths) for unknown_path in NicePath( setup_path if generate_setup else package_path).walk(valid_paths): unknown_path.unlink() logger.debug(f"Deleted {NicePath(unknown_path)}")
def write_master_package(package: MasterPackage, output_path: Path, generate_setup: bool) -> None: """ Create mypy-boto3 stubs. Arguments: package -- Master package. output_path -- Path to output folder. generate_setup -- Generate ready-to-install or to-use package. """ logger = get_logger() setup_path = output_path / "master_package" if generate_setup: package_path = setup_path / package.name else: package_path = output_path / package.name package_path.mkdir(exist_ok=True, parents=True) templates_path = Path("master") module_templates_path = templates_path / "master" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend( [ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ] ) file_paths.extend( [ (package_path / "__init__.py", module_templates_path / "__init__.py.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), (package_path / "main.py", module_templates_path / "main.py.jinja2"), (package_path / "boto3_init.py", module_templates_path / "boto3_init.py.jinja2"), (package_path / "boto3_session.py", module_templates_path / "boto3_session.py.jinja2"), ( package_path / "boto3_init_gen.py", module_templates_path / "boto3_init_stub.py.jinja2", ), ( package_path / "boto3_session_gen.py", module_templates_path / "boto3_session_stub.py.jinja2", ), ( package_path / "boto3_init_stub.py", module_templates_path / "boto3_init_stub.py.jinja2", ), ( package_path / "boto3_session_stub.py", module_templates_path / "boto3_session_stub.py.jinja2", ), (package_path / "submodules.py", module_templates_path / "submodules.py.jinja2"), ] ) for file_path, template_path in file_paths: content = render_jinja2_template(template_path, package=package) if file_path.suffix in [".py", ".pyi"]: content = sort_imports(content, "mypy_boto3", extension=file_path.suffix[1:]) content = blackify(content, file_path) if file_path.suffix == ".md": content = insert_md_toc(content) content = fix_pypi_headers(content) content = format_md(content) if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) logger.debug(f"Updated {NicePath(file_path)}") valid_paths = dict(file_paths).keys() for unknown_path in NicePath(setup_path if generate_setup else package_path).walk(valid_paths): unknown_path.unlink() logger.debug(f"Deleted {NicePath(unknown_path)}")