def write_service_package(package: ServicePackage, output_path: Path) -> List[Path]: modified_paths: List[Path] = [] package_path = output_path / package.name output_path.mkdir(exist_ok=True) package_path.mkdir(exist_ok=True) templates_path = Path("service") module_templates_path = templates_path / "service" file_paths = [ (output_path / "setup.py", templates_path / "setup.py.jinja2"), (output_path / "README.md", templates_path / "README.md.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), (package_path / "__init__.py", module_templates_path / "__init__.py.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), ( package_path / ServiceModuleName.client.file_name, module_templates_path / ServiceModuleName.client.template_name, ), ] if package.service_resource: file_paths.append(( package_path / ServiceModuleName.service_resource.file_name, module_templates_path / ServiceModuleName.service_resource.template_name, )) if package.paginators: file_paths.append(( package_path / ServiceModuleName.paginator.file_name, module_templates_path / ServiceModuleName.paginator.template_name, )) if package.waiters: file_paths.append(( package_path / ServiceModuleName.waiter.file_name, module_templates_path / ServiceModuleName.waiter.template_name, )) if package.typed_dicts: file_paths.append(( package_path / ServiceModuleName.type_defs.file_name, module_templates_path / ServiceModuleName.type_defs.template_name, )) for file_path, template_path in file_paths: content = render_jinja2_template( template_path, package=package, service_name=package.service_name, ) content = blackify(content, file_path) if not file_path.exists() or file_path.read_text() != content: modified_paths.append(file_path) file_path.write_text(content) return modified_paths
def test_render_jinja2_template(self, JinjaManagerMock: MagicMock, TEMPLATES_PATH_MOCK: MagicMock) -> None: template_path_mock = MagicMock() result = render_jinja2_template(template_path_mock, "package", "service_name") JinjaManagerMock.get_environment.assert_called_with() JinjaManagerMock.get_environment().get_template.assert_called_with( template_path_mock.as_posix()) JinjaManagerMock.get_environment().get_template( ).render.assert_called_with(package="package", service_name="service_name") assert result == JinjaManagerMock.get_environment().get_template( ).render() TEMPLATES_PATH_MOCK.__truediv__().exists.return_value = False with pytest.raises(ValueError): render_jinja2_template(template_path_mock, "package", "service_name")
def write_service_docs(package: ServicePackage, output_path: Path) -> None: """ Create service docs files. Arguments: package -- Service package. output_path -- Path to output folder. """ logger = get_logger() docs_path = output_path / f"{package.service_name.module_name}" docs_path.mkdir(exist_ok=True) templates_path = Path("service_docs") file_paths = [ (docs_path / "README.md", templates_path / "README.md.jinja2"), (docs_path / "client.md", templates_path / "client.md.jinja2"), ] if package.literals: file_paths.append( (docs_path / "literals.md", templates_path / "literals.md.jinja2")) if package.typed_dicts: file_paths.append((docs_path / "type_defs.md", templates_path / "type_defs.md.jinja2")) if package.waiters: file_paths.append( (docs_path / "waiters.md", templates_path / "waiters.md.jinja2")) if package.paginators: file_paths.append((docs_path / "paginators.md", templates_path / "paginators.md.jinja2")) if package.service_resource: file_paths.append((docs_path / "service_resource.md", templates_path / "service_resource.md.jinja2")) for file_path, template_path in file_paths: content = render_jinja2_template( template_path, package=package, service_name=package.service_name, ) content = insert_md_toc(content) content = format_md(content) if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) logger.debug(f"Updated {NicePath(file_path)}") valid_paths = dict(file_paths).keys() for unknown_path in NicePath(docs_path).walk(valid_paths): unknown_path.unlink() logger.debug(f"Deleted {NicePath(unknown_path)}")
def write_master_package(package: MasterPackage, output_path: Path) -> List[Path]: modified_paths: List[Path] = [] package_path = output_path / package.name output_path.mkdir(exist_ok=True) package_path.mkdir(exist_ok=True) templates_path = Path("master") module_templates_path = templates_path / "master" file_paths = [ (output_path / "setup.py", templates_path / "setup.py.jinja2"), (output_path / "README.md", templates_path / "README.md.jinja2"), (package_path / "__init__.py", module_templates_path / "__init__.py.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), (package_path / "main.py", module_templates_path / "main.py.jinja2"), (package_path / "boto3_init.py", module_templates_path / "boto3_init.py.jinja2"), (package_path / "boto3_session.py", module_templates_path / "boto3_session.py.jinja2"), (package_path / "boto3_init_gen.py", module_templates_path / "boto3_init_stub.py.jinja2"), ( package_path / "boto3_session_gen.py", module_templates_path / "boto3_session_stub.py.jinja2", ), (package_path / "boto3_init_stub.py", module_templates_path / "boto3_init_stub.py.jinja2"), ( package_path / "boto3_session_stub.py", module_templates_path / "boto3_session_stub.py.jinja2", ), (package_path / "submodules.py", module_templates_path / "submodules.py.jinja2"), ] for file_path, template_path in file_paths: content = render_jinja2_template(template_path, package=package) content = blackify(content, file_path) if not file_path.exists() or file_path.read_text() != content: modified_paths.append(file_path) file_path.write_text(content) return modified_paths
def test_render_jinja2_template(self, JinjaManagerMock: MagicMock, TEMPLATES_PATH_MOCK: MagicMock) -> None: template_path_mock = MagicMock() package_mock = MagicMock() service_name_mock = MagicMock() result = render_jinja2_template(template_path_mock, package_mock, service_name_mock) JinjaManagerMock.get_environment.assert_called_with() JinjaManagerMock.get_environment().get_template.assert_called() JinjaManagerMock.get_environment().get_template( ).render.assert_called_with(package=package_mock, service_name=service_name_mock) assert result == JinjaManagerMock.get_environment().get_template( ).render()
def write_boto3_stubs_package(package: Boto3StubsPackage, output_path: Path) -> List[Path]: modified_paths: List[Path] = [] package_path = output_path / package.name output_path.mkdir(exist_ok=True) package_path.mkdir(exist_ok=True) templates_path = Path("boto3-stubs") module_templates_path = templates_path / "boto3-stubs" file_paths = [ (output_path / "setup.py", templates_path / "setup.py.jinja2"), (output_path / "README.md", templates_path / "README.md.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), (package_path / "__init__.pyi", module_templates_path / "__init__.pyi.jinja2"), (package_path / "session.pyi", module_templates_path / "session.pyi.jinja2"), (package_path / "__init__.py", module_templates_path / "__init__.py.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), ] for file_path, template_path in file_paths: content = render_jinja2_template(template_path, package=package) content = blackify(content, file_path) if not file_path.exists() or file_path.read_text() != content: modified_paths.append(file_path) file_path.write_text(content) for static_path in BOTO3_STUBS_STATIC_PATH.glob("**/*.pyi"): relative_output_path = static_path.relative_to(BOTO3_STUBS_STATIC_PATH) file_path = package_path / relative_output_path file_path.parent.mkdir(exist_ok=True) if file_path.exists() and filecmp.cmp(static_path.as_posix(), file_path.as_posix()): continue shutil.copy(static_path, file_path) modified_paths.append(file_path) return modified_paths
def write_docs(self, package: Package, templates_path: Path) -> None: """ Generate docs for a package. """ self.output_path.mkdir(exist_ok=True, parents=True) file_paths = [] for template_path in templates_path.glob("**/*.jinja2"): file_name = template_path.name.rsplit(".", 1)[0] file_paths.append((self.output_path / file_name, template_path)) for file_path, template_path in file_paths: content = render_jinja2_template( template_path, package=package, ) content = insert_md_toc(content) content = format_md(content) if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) self.logger.debug(f"Updated {NicePath(file_path)}")
def _render_templates( self, package: Package, file_paths: list[tuple[Path, Path]], service_name: ServiceName | None = None, ) -> None: for file_path, template_path in file_paths: content = render_jinja2_template( template_path, package=package, service_name=service_name, ) if file_path.suffix in [".py", ".pyi"]: content = sort_imports( content, package.name, extension="pyi", third_party=[ "boto3", "botocore", "aiobotocore", *([ package.data.get_service_package_name(i) for i in package.service_names ] if not service_name else [ package.data.get_service_package_name(service_name) ]), ], ) content = blackify(content, file_path) if file_path.suffix == ".md": content = insert_md_toc(content) content = fix_pypi_headers(content) content = format_md(content) if not file_path.parent.exists(): file_path.parent.mkdir(exist_ok=True, parents=True) if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) self.logger.debug(f"Rendered {NicePath(file_path)}")
def write_service_package(package: ServicePackage, output_path: Path, generate_setup: bool) -> None: """ Create stubs files for service. Arguments: package -- Service package. output_path -- Path to output folder. generate_setup -- Generate ready-to-install or to-use package. """ logger = get_logger() setup_path = output_path / f"{package.service_name.module_name}_package" if generate_setup: package_path = setup_path / package.name else: package_path = output_path / package.name package_path.mkdir(exist_ok=True, parents=True) templates_path = Path("service") module_templates_path = templates_path / "service" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend([ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ]) file_paths.extend([ (package_path / "version.py", module_templates_path / "version.py.jinja2"), (package_path / "__init__.pyi", module_templates_path / "__init__.pyi.jinja2"), (package_path / "__init__.py", module_templates_path / "__init__.pyi.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), ( package_path / ServiceModuleName.client.stub_file_name, module_templates_path / ServiceModuleName.client.template_name, ), ( package_path / ServiceModuleName.client.file_name, module_templates_path / ServiceModuleName.client.template_name, ), ]) if package.service_resource: file_paths.extend(( ( package_path / ServiceModuleName.service_resource.stub_file_name, module_templates_path / ServiceModuleName.service_resource.template_name, ), ( package_path / ServiceModuleName.service_resource.file_name, module_templates_path / ServiceModuleName.service_resource.template_name, ), )) if package.paginators: file_paths.extend(( ( package_path / ServiceModuleName.paginator.stub_file_name, module_templates_path / ServiceModuleName.paginator.template_name, ), ( package_path / ServiceModuleName.paginator.file_name, module_templates_path / ServiceModuleName.paginator.template_name, ), )) if package.waiters: file_paths.extend(( ( package_path / ServiceModuleName.waiter.stub_file_name, module_templates_path / ServiceModuleName.waiter.template_name, ), ( package_path / ServiceModuleName.waiter.file_name, module_templates_path / ServiceModuleName.waiter.template_name, ), )) if package.literals: file_paths.extend(( ( package_path / ServiceModuleName.literals.stub_file_name, module_templates_path / ServiceModuleName.literals.template_name, ), ( package_path / ServiceModuleName.literals.file_name, module_templates_path / ServiceModuleName.literals.template_name, ), )) if package.typed_dicts: file_paths.extend(( ( package_path / ServiceModuleName.type_defs.stub_file_name, module_templates_path / ServiceModuleName.type_defs.template_name, ), ( package_path / ServiceModuleName.type_defs.file_name, module_templates_path / ServiceModuleName.type_defs.template_name, ), )) for file_path, template_path in file_paths: content = render_jinja2_template( template_path, package=package, service_name=package.service_name, ) if file_path.suffix in [".py", ".pyi"]: content = sort_imports(content, package.service_name.module_name, extension="pyi") content = blackify(content, file_path) if file_path.suffix == ".md": content = insert_md_toc(content) content = fix_pypi_headers(content) content = format_md(content) if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) logger.debug(f"Updated {NicePath(file_path)}") valid_paths = dict(file_paths).keys() for unknown_path in NicePath( setup_path if generate_setup else package_path).walk(valid_paths): unknown_path.unlink() logger.debug(f"Deleted {NicePath(unknown_path)}")
def write_boto3_stubs_package(package: Boto3StubsPackage, output_path: Path, generate_setup: bool) -> List[Path]: setup_path = output_path / "boto3_stubs_package" if not generate_setup: setup_path = output_path modified_paths: List[Path] = [] package_path = setup_path / package.name if not generate_setup: package_path = setup_path / "boto3" if setup_path.exists(): shutil.rmtree(setup_path) setup_path.mkdir(exist_ok=True) package_path.mkdir(exist_ok=True) templates_path = Path("boto3-stubs") module_templates_path = templates_path / "boto3-stubs" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend([ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ]) file_paths.extend([ (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), (package_path / "__init__.pyi", module_templates_path / "__init__.pyi.jinja2"), (package_path / "__init__.py", module_templates_path / "__init__.pyi.jinja2"), (package_path / "session.pyi", module_templates_path / "session.pyi.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), ]) for file_path, template_path in file_paths: content = render_jinja2_template(template_path, package=package) content = blackify(content, file_path) content = sort_imports( content, "boto3_stubs", extension="pyi", third_party=[ "boto3", "botocore", *[i.module_name for i in package.service_names] ], ) if not file_path.exists() or file_path.read_text() != content: modified_paths.append(file_path) file_path.write_text(content) for static_path in BOTO3_STUBS_STATIC_PATH.glob("**/*.pyi"): relative_output_path = static_path.relative_to(BOTO3_STUBS_STATIC_PATH) file_path = package_path / relative_output_path file_path.parent.mkdir(exist_ok=True) if file_path.exists() and filecmp.cmp(static_path.as_posix(), file_path.as_posix()): continue shutil.copy(static_path, file_path) modified_paths.append(file_path) return modified_paths
def write_master_package(package: MasterPackage, output_path: Path, generate_setup: bool) -> List[Path]: setup_path = output_path / "master_package" if not generate_setup: setup_path = output_path modified_paths: List[Path] = [] package_path = setup_path / package.name if setup_path.exists(): shutil.rmtree(setup_path) setup_path.mkdir(exist_ok=True) package_path.mkdir(exist_ok=True) templates_path = Path("master") module_templates_path = templates_path / "master" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend([ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ]) file_paths.extend([ (package_path / "__init__.py", module_templates_path / "__init__.py.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), (package_path / "main.py", module_templates_path / "main.py.jinja2"), (package_path / "boto3_init.py", module_templates_path / "boto3_init.py.jinja2"), (package_path / "boto3_session.py", module_templates_path / "boto3_session.py.jinja2"), ( package_path / "boto3_init_gen.py", module_templates_path / "boto3_init_stub.py.jinja2", ), ( package_path / "boto3_session_gen.py", module_templates_path / "boto3_session_stub.py.jinja2", ), ( package_path / "boto3_init_stub.py", module_templates_path / "boto3_init_stub.py.jinja2", ), ( package_path / "boto3_session_stub.py", module_templates_path / "boto3_session_stub.py.jinja2", ), (package_path / "submodules.py", module_templates_path / "submodules.py.jinja2"), ]) for file_path, template_path in file_paths: content = render_jinja2_template(template_path, package=package) content = blackify(content, file_path) content = sort_imports(content, "mypy_boto3", extension=file_path.suffix[1:]) if not file_path.exists() or file_path.read_text() != content: modified_paths.append(file_path) file_path.write_text(content) return modified_paths
def write_service_package( package: ServicePackage, output_path: Path, generate_setup: bool ) -> List[Path]: setup_path = output_path / f"{package.service_name.module_name}_package" if not generate_setup: setup_path = output_path modified_paths: List[Path] = [] package_path = setup_path / package.name if setup_path.exists(): shutil.rmtree(setup_path) setup_path.mkdir(exist_ok=True) package_path.mkdir(exist_ok=True) templates_path = Path("service") module_templates_path = templates_path / "service" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend( [ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ] ) file_paths.extend( [ (package_path / "version.py", module_templates_path / "version.py.jinja2"), (package_path / "__init__.pyi", module_templates_path / "__init__.pyi.jinja2"), (package_path / "__init__.py", module_templates_path / "__init__.pyi.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), ( package_path / ServiceModuleName.client.stub_file_name, module_templates_path / ServiceModuleName.client.template_name, ), ( package_path / ServiceModuleName.client.file_name, module_templates_path / ServiceModuleName.client.template_name, ), ] ) if package.service_resource: file_paths.extend( ( ( package_path / ServiceModuleName.service_resource.stub_file_name, module_templates_path / ServiceModuleName.service_resource.template_name, ), ( package_path / ServiceModuleName.service_resource.file_name, module_templates_path / ServiceModuleName.service_resource.template_name, ), ) ) if package.paginators: file_paths.extend( ( ( package_path / ServiceModuleName.paginator.stub_file_name, module_templates_path / ServiceModuleName.paginator.template_name, ), ( package_path / ServiceModuleName.paginator.file_name, module_templates_path / ServiceModuleName.paginator.template_name, ), ) ) if package.waiters: file_paths.extend( ( ( package_path / ServiceModuleName.waiter.stub_file_name, module_templates_path / ServiceModuleName.waiter.template_name, ), ( package_path / ServiceModuleName.waiter.file_name, module_templates_path / ServiceModuleName.waiter.template_name, ), ) ) if package.typed_dicts: file_paths.extend( ( ( package_path / ServiceModuleName.type_defs.stub_file_name, module_templates_path / ServiceModuleName.type_defs.template_name, ), ( package_path / ServiceModuleName.type_defs.file_name, module_templates_path / ServiceModuleName.type_defs.template_name, ), ) ) for file_path, template_path in file_paths: content = render_jinja2_template( template_path, package=package, service_name=package.service_name, ) content = blackify(content, file_path) content = sort_imports(content, package.service_name.module_name, extension="pyi") if not file_path.exists() or file_path.read_text() != content: modified_paths.append(file_path) file_path.write_text(content) return modified_paths
def write_botocore_stubs_package(output_path: Path, generate_setup: bool) -> None: """ Generate botocore-stubs stub files. Arguments: output_path -- Path to output folder. generate_setup -- Generate ready-to-install or to-use package. """ logger = get_logger() setup_path = output_path / "botocore_stubs_package" if generate_setup: package_path = setup_path / BOTOCORE_STUBS_NAME else: package_path = output_path / "botocore" package_path.mkdir(exist_ok=True, parents=True) templates_path = Path("botocore-stubs") module_templates_path = templates_path / "botocore-stubs" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend([ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ]) file_paths.extend([ (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), ]) for file_path, template_path in file_paths: content = render_jinja2_template(template_path) if file_path.suffix in [".py", ".pyi"]: content = sort_imports( content, "boto3_stubs", extension="pyi", ) content = blackify(content, file_path) if file_path.suffix == ".md": content = insert_md_toc(content) content = fix_pypi_headers(content) content = format_md(content) if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) logger.debug(f"Updated {NicePath(file_path)}") static_paths = [] for static_path in BOTOCORE_STUBS_STATIC_PATH.glob("**/*.pyi"): relative_output_path = static_path.relative_to( BOTOCORE_STUBS_STATIC_PATH) file_path = package_path / relative_output_path static_paths.append(file_path) file_path.parent.mkdir(exist_ok=True) content = static_path.read_text() if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) logger.debug(f"Updated {NicePath(file_path)}") valid_paths = (*dict(file_paths).keys(), *static_paths) for unknown_path in NicePath( setup_path if generate_setup else package_path).walk(valid_paths): unknown_path.unlink() logger.debug(f"Deleted {NicePath(unknown_path)}")
def write_master_package(package: MasterPackage, output_path: Path, generate_setup: bool) -> None: """ Create mypy-boto3 stubs. Arguments: package -- Master package. output_path -- Path to output folder. generate_setup -- Generate ready-to-install or to-use package. """ logger = get_logger() setup_path = output_path / "master_package" if generate_setup: package_path = setup_path / package.name else: package_path = output_path / package.name package_path.mkdir(exist_ok=True, parents=True) templates_path = Path("master") module_templates_path = templates_path / "master" file_paths: List[Tuple[Path, Path]] = [] if generate_setup: file_paths.extend( [ (setup_path / "setup.py", templates_path / "setup.py.jinja2"), (setup_path / "README.md", templates_path / "README.md.jinja2"), ] ) file_paths.extend( [ (package_path / "__init__.py", module_templates_path / "__init__.py.jinja2"), (package_path / "__main__.py", module_templates_path / "__main__.py.jinja2"), (package_path / "py.typed", module_templates_path / "py.typed.jinja2"), (package_path / "version.py", module_templates_path / "version.py.jinja2"), (package_path / "main.py", module_templates_path / "main.py.jinja2"), (package_path / "boto3_init.py", module_templates_path / "boto3_init.py.jinja2"), (package_path / "boto3_session.py", module_templates_path / "boto3_session.py.jinja2"), ( package_path / "boto3_init_gen.py", module_templates_path / "boto3_init_stub.py.jinja2", ), ( package_path / "boto3_session_gen.py", module_templates_path / "boto3_session_stub.py.jinja2", ), ( package_path / "boto3_init_stub.py", module_templates_path / "boto3_init_stub.py.jinja2", ), ( package_path / "boto3_session_stub.py", module_templates_path / "boto3_session_stub.py.jinja2", ), (package_path / "submodules.py", module_templates_path / "submodules.py.jinja2"), ] ) for file_path, template_path in file_paths: content = render_jinja2_template(template_path, package=package) if file_path.suffix in [".py", ".pyi"]: content = sort_imports(content, "mypy_boto3", extension=file_path.suffix[1:]) content = blackify(content, file_path) if file_path.suffix == ".md": content = insert_md_toc(content) content = fix_pypi_headers(content) content = format_md(content) if not file_path.exists() or file_path.read_text() != content: file_path.write_text(content) logger.debug(f"Updated {NicePath(file_path)}") valid_paths = dict(file_paths).keys() for unknown_path in NicePath(setup_path if generate_setup else package_path).walk(valid_paths): unknown_path.unlink() logger.debug(f"Deleted {NicePath(unknown_path)}")