def test_parse_pep420_namespace_package() -> None:
metadata = Metadata(FIXTURES /
"projects/demo-pep420-package/pyproject.toml")
paths = metadata.convert_package_paths()
assert paths["package_dir"] == {}
assert paths["packages"] == ["foo.my_package"]
assert paths["py_modules"] == []
def test_parse_src_package_by_include():
metadata = Metadata(FIXTURES /
"projects/demo-src-package-include/pyproject.toml")
paths = metadata.convert_package_paths()
assert paths["package_dir"] == {}
assert paths["packages"] == ["sub.my_package"]
assert paths["py_modules"] == []
def test_explicit_package_dir() -> None:
metadata = Metadata(FIXTURES /
"projects/demo-explicit-package-dir/pyproject.toml")
paths = metadata.convert_package_paths()
assert paths["packages"] == ["my_package"]
assert paths["py_modules"] == []
assert paths["package_dir"] == {"": "foo"}
def test_package_with_old_include() -> None:
metadata = Metadata(FIXTURES /
"projects/demo-package-include-old/pyproject.toml")
paths = metadata.convert_package_paths()
assert paths["py_modules"] == []
assert paths["packages"] == ["my_package"]
assert paths["package_dir"] == {}
assert paths["package_data"] == {"": ["*"]}
assert not metadata.classifiers
def test_parse_module() -> None:
metadata = Metadata(FIXTURES / "projects/demo-module/pyproject.toml")
assert metadata.name == "demo-module"
assert metadata.version == "0.1.0"
assert metadata.author == ""
assert metadata.author_email == "frostming <*****@*****.**>"
paths = metadata.convert_package_paths()
assert sorted(paths["py_modules"]) == ["bar_module", "foo_module"]
assert paths["packages"] == []
assert paths["package_dir"] == {}
def test_autogen_classifiers():
metadata = Metadata(FIXTURES / "projects/demo-module/pyproject.toml")
classifiers = metadata.classifiers
for python_version in ("3", "3.5", "3.6", "3.7", "3.8", "3.9"):
assert f"Programming Language :: Python :: {python_version}" in classifiers
assert "Programming Language :: Python :: 2.7" not in classifiers
assert "License :: OSI Approved :: MIT License" in classifiers
def test_project_name_and_version_missing() -> None:
metadata = Metadata(FIXTURES /
"projects/demo-no-name-nor-version/pyproject.toml")
assert metadata.version is None
assert metadata.name is None
assert metadata.project_name is None
assert metadata.project_filename == "UNKNOWN"
def test_project_version_use_scm(project_with_scm) -> None:
metadata = Metadata(project_with_scm / "pyproject.toml")
assert metadata.version == "0.1.0"
project_with_scm.joinpath("test.txt").write_text("hello\n")
subprocess.check_call(["git", "add", "test.txt"])
date = datetime.utcnow().strftime("%Y%m%d")
assert metadata.version == f"0.1.0+d{date}"
subprocess.check_call(["git", "commit", "-m", "add test.txt"])
assert "0.1.1.dev1+g" in metadata.version
def test_parse_package_with_extras() -> None:
metadata = Metadata(FIXTURES /
"projects/demo-combined-extras/pyproject.toml")
assert metadata.dependencies == ["urllib3"]
assert metadata.optional_dependencies == {
"be": ["idna"],
"te": ["chardet"],
"all": ["idna", "chardet"],
}
assert metadata.requires_extra == {
"be": ['idna; extra == "be"'],
"te": ['chardet; extra == "te"'],
"all": ['idna; extra == "all"', 'chardet; extra == "all"'],
}
def read_pyproject_toml(self, filepath):
from pdm.pep517.metadata import Metadata
try:
metadata = Metadata(filepath)
except ValueError:
return {}
return {
"name": metadata.name,
"version": metadata.version,
"install_requires": metadata.dependencies,
"extras_require": metadata.optional_dependencies,
"python_requires": metadata.requires_python,
}
def get_requires_for_build_wheel(
config_settings: Optional[Mapping[str, Any]] = None) -> List[str]:
"""
Returns an additional list of requirements for building, as PEP508 strings,
above and beyond those specified in the pyproject.toml file.
When C-extension build is needed, setuptools should be required, otherwise
just return an empty list.
"""
meta = Metadata(Path("pyproject.toml"))
if meta.build:
return ["setuptools>=40.8.0"]
else:
return []
def meta(self) -> Metadata:
if not self._meta:
self._meta = Metadata(self.location / "pyproject.toml")
return self._meta
class Builder:
"""Base class for building and distributing a package from given path."""
DEFAULT_EXCLUDES = ["ez_setup", "*__pycache__", "tests", "tests.*"]
def __init__(self, location: Union[str, Path]) -> None:
self._old_cwd = None
self.location = Path(location).absolute()
self._meta = None
@property
def meta(self) -> Metadata:
if not self._meta:
self._meta = Metadata(self.location / "pyproject.toml")
# Open the validation for next release
self._meta.validate(False)
return self._meta
def __enter__(self) -> "Builder":
self._old_cwd = os.getcwd()
os.chdir(self.location)
return self
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
os.chdir(self._old_cwd)
def build(self, build_dir: str, **kwargs) -> str:
raise NotImplementedError
def _find_files_iter(self, for_sdist: bool = False) -> Iterator[str]:
includes = set()
find_froms = set()
excludes = set()
dont_find_froms = set()
source_includes = self.meta.source_includes or ["tests"]
meta_includes = itertools.chain(self.meta.includes, source_includes)
meta_excludes = list(self.meta.excludes)
if not for_sdist:
# exclude source-includes for non-sdist builds
meta_excludes.extend(source_includes)
for pat in meta_includes:
if os.path.basename(pat) == "*":
pat = pat[:-2]
if "*" in pat or os.path.isfile(pat):
includes.add(pat)
else:
find_froms.add(pat)
if not self.meta.includes:
top_packages = _find_top_packages(self.meta.package_dir or ".")
if top_packages:
find_froms.update(top_packages)
else:
includes.add(f"{self.meta.package_dir or '.'}/*.py")
for pat in meta_excludes:
if "*" in pat or os.path.isfile(pat):
excludes.add(pat)
else:
dont_find_froms.add(pat)
include_globs = {
path: key
for key in includes for path in glob.glob(key)
}
excludes_globs = {
path: key
for key in excludes for path in glob.glob(key)
}
includes, excludes = _merge_globs(include_globs, excludes_globs)
for path in find_froms:
if any(_match_path(path, item) for item in dont_find_froms):
continue
path_base = os.path.dirname(path)
if not path_base or path_base == ".":
# the path is top level itself
path_base = path
for root, dirs, filenames in os.walk(path):
for filename in filenames:
if filename.endswith(".pyc") or any(
_match_path(os.path.join(root, filename), item)
for item in excludes):
continue
yield os.path.join(root, filename)
for path in includes:
if os.path.isfile(path):
yield path
if not for_sdist:
return
if self.meta.build and os.path.isfile(self.meta.build):
yield self.meta.build
for pat in ("COPYING", "LICENSE"):
for path in glob.glob(pat + "*"):
if os.path.isfile(path):
yield path
if self.meta.readme and os.path.isfile(self.meta.readme):
yield self.meta.readme
if self.meta.filepath.exists():
yield "pyproject.toml"
def find_files_to_add(self, for_sdist: bool = False) -> List[Path]:
"""Traverse the project path and return a list of file names
that should be included in a sdist distribution.
If for_sdist is True, will include files like LICENSE, README and pyproject
Produce a paths list relative to the source dir.
"""
return sorted(set(Path(p) for p in self._find_files_iter(for_sdist)))
def format_setup_py(self) -> str:
before, extra, after = [], [], []
meta = self.meta
kwargs = {
"name": meta.name,
"version": meta.version,
"author": meta.author,
"license": meta.license_type,
"author_email": meta.author_email,
"maintainer": meta.maintainer,
"maintainer_email": meta.maintainer_email,
"description": meta.description,
"url": (meta.project_urls or {}).get("homepage", ""),
}
if meta.build:
# The build script must contain a `build(setup_kwargs)`, we just import
# and execute it.
after.extend([
"from {} import build\n".format(meta.build.split(".")[0]),
"build(setup_kwargs)\n",
])
package_paths = meta.convert_package_paths()
if package_paths["packages"]:
extra.append(" 'packages': {},\n".format(
_format_list(package_paths["packages"], 8)))
if package_paths["package_dir"]:
extra.append(" 'package_dir': {!r},\n".format(
package_paths["package_dir"]))
if package_paths["package_data"]:
extra.append(" 'package_data': {!r},\n".format(
package_paths["package_data"]))
if package_paths["exclude_package_data"]:
extra.append(" 'exclude_package_data': {!r},\n".format(
package_paths["exclude_package_data"]))
if meta.readme:
before.append(OPEN_README.format(readme=meta.readme))
elif meta.long_description:
before.append("long_description = '''{}'''\n".format(
repr(meta.long_description)[1:-1]))
else:
before.append("long_description = None\n")
if meta.long_description_content_type:
extra.append(" 'long_description_content_type': {!r},\n".format(
meta.long_description_content_type))
if meta.keywords:
extra.append(" 'keywords': {!r},\n".format(meta.keywords))
if meta.classifiers:
extra.append(" 'classifiers': {},\n".format(
_format_list(meta.classifiers, 8)))
if meta.dependencies:
before.append("INSTALL_REQUIRES = {}\n".format(
_format_list(meta.dependencies)))
extra.append(" 'install_requires': INSTALL_REQUIRES,\n")
if meta.optional_dependencies:
before.append("EXTRAS_REQUIRE = {}\n".format(
_format_dict_list(meta.optional_dependencies)))
extra.append(" 'extras_require': EXTRAS_REQUIRE,\n")
if meta.requires_python:
extra.append(" 'python_requires': {!r},\n".format(
meta.requires_python))
if meta.entry_points:
before.append("ENTRY_POINTS = {}\n".format(
_format_dict_list(meta.entry_points)))
extra.append(" 'entry_points': ENTRY_POINTS,\n")
return SETUP_FORMAT.format(before="".join(before),
after="".join(after),
extra="".join(extra),
**kwargs)
def format_pkginfo(self, full=True) -> str:
meta = self.meta
content = METADATA_BASE.format(
name=meta.name or "UNKNOWN",
version=meta.version or "UNKNOWN",
license=meta.license_type or "UNKNOWN",
description=meta.description or "UNKNOWN",
)
# Optional fields
if meta.keywords:
content += "Keywords: {}\n".format(",".join(meta.keywords))
if meta.author:
content += "Author: {}\n".format(meta.author)
if meta.author_email:
content += "Author-email: {}\n".format(meta.author_email)
if meta.maintainer:
content += "Maintainer: {}\n".format(meta.maintainer)
if meta.maintainer_email:
content += "Maintainer-email: {}\n".format(meta.maintainer_email)
if meta.requires_python:
content += "Requires-Python: {}\n".format(meta.requires_python)
for classifier in meta.classifiers or []:
content += "Classifier: {}\n".format(classifier)
if full:
for dep in sorted(meta.dependencies):
content += "Requires-Dist: {}\n".format(dep)
for extra, reqs in sorted(meta.requires_extra.items()):
content += "Provides-Extra: {}\n".format(extra)
if full:
for dep in reqs:
content += "Requires-Dist: {}\n".format(dep)
for url in sorted(meta.project_urls or {}):
content += "Project-URL: {}, {}\n".format(url,
meta.project_urls[url])
if meta.long_description_content_type:
content += "Description-Content-Type: {}\n".format(
meta.long_description_content_type)
if meta.long_description:
readme = meta.long_description
if full:
content += "\n" + readme + "\n"
else:
content += "Description: {}\n".format(
textwrap.indent(readme, " " * 8,
lambda line: True).lstrip())
return content
def ensure_setup_py(self, clean: bool = True) -> Path:
"""Ensures the requirement has a setup.py ready."""
# XXX: Currently only handle PDM project, and do nothing if not.
setup_py_path = self.location.joinpath("setup.py")
if setup_py_path.is_file():
return setup_py_path
setup_py_path.write_text(self.format_setup_py(), encoding="utf-8")
# Clean this temp file when process exits
def cleanup():
try:
setup_py_path.unlink()
except OSError:
pass
if clean:
atexit.register(cleanup)
return setup_py_path
def test_parse_src_package() -> None:
metadata = Metadata(FIXTURES / "projects/demo-src-package/pyproject.toml")
paths = metadata.convert_package_paths()
assert paths["packages"] == ["my_package"]
assert paths["py_modules"] == []
assert paths["package_dir"] == {"": "src"}
def test_parse_error_package() -> None:
metadata = Metadata(FIXTURES /
"projects/demo-package-include-error/pyproject.toml")
with pytest.raises(ValueError):
metadata.convert_package_paths()
def _get_current_version():
from pdm.pep517.metadata import Metadata
metadata = Metadata(PROJECT_DIR / "pyproject.toml")
return metadata.version
def test_src_dir_containing_modules() -> None:
metadata = Metadata(FIXTURES / "projects/demo-src-pymodule/pyproject.toml")
paths = metadata.convert_package_paths()
assert paths["package_dir"] == {"": "src"}
assert not paths["packages"]
assert paths["py_modules"] == ["foo_module"]
def meta(self) -> Metadata:
if not self._meta:
self._meta = Metadata(self.location / "pyproject.toml")
# Open the validation for next release
self._meta.validate(False)
return self._meta
class Builder:
"""Base class for building and distributing a package from given path."""
DEFAULT_EXCLUDES = ["build"]
def __init__(
self,
location: Union[str, Path],
config_settings: Optional[Mapping[str, Any]] = None,
) -> None:
self._old_cwd: Optional[str] = None
self.location = Path(location).absolute()
self.config_settings = config_settings
self._meta: Optional[Metadata] = None
@property
def meta(self) -> Metadata:
if not self._meta:
self._meta = Metadata(self.location / "pyproject.toml")
# Open the validation for next release
self._meta.validate(False)
return self._meta
@property
def meta_version(self) -> str:
meta_version = self.meta.version
if meta_version is None:
return "0.0.0"
return to_filename(safe_version(meta_version))
def __enter__(self: T) -> T:
self._old_cwd = os.getcwd()
os.chdir(self.location)
return self
def __exit__(self, *args: Any) -> None:
assert self._old_cwd
os.chdir(self._old_cwd)
def build(self, build_dir: str, **kwargs: Any) -> str:
raise NotImplementedError
def _get_include_and_exclude_paths(self,
for_sdist: bool = False
) -> Tuple[List[str], List[str]]:
includes = set()
excludes = set(self.DEFAULT_EXCLUDES)
meta_excludes = list(self.meta.excludes)
source_includes = self.meta.source_includes or ["tests"]
if not for_sdist:
# exclude source-includes for non-sdist builds
meta_excludes.extend(source_includes)
if not self.meta.includes:
top_packages = _find_top_packages(self.meta.package_dir or ".")
if top_packages:
includes.update(top_packages)
else:
includes.add(f"{self.meta.package_dir or '.'}/*.py")
else:
includes.update(self.meta.includes)
includes.update(source_includes)
excludes.update(meta_excludes)
include_globs = {
os.path.normpath(path): key
for key in includes for path in glob.iglob(key, recursive=True)
}
excludes_globs = {
os.path.normpath(path): key
for key in excludes for path in glob.iglob(key, recursive=True)
}
include_paths, exclude_paths = _merge_globs(include_globs,
excludes_globs)
return sorted(include_paths), sorted(exclude_paths)
def _is_excluded(self, path: str, exclude_paths: List[str]) -> bool:
return any(
is_same_or_descendant_path(path, exclude_path)
for exclude_path in exclude_paths)
def _find_files_iter(self, for_sdist: bool = False) -> Iterator[str]:
includes, excludes = self._get_include_and_exclude_paths(for_sdist)
for include_path in includes:
path = Path(include_path)
if path.is_file():
yield include_path
continue
# The path is a directory name
for path in path.glob("**/*"):
if not path.is_file():
continue
rel_path = path.absolute().relative_to(
self.location).as_posix()
if path.name.endswith(".pyc") or self._is_excluded(
rel_path, excludes):
continue
yield rel_path
if not for_sdist:
return
if self.meta.build and os.path.isfile(self.meta.build):
yield self.meta.build
for pat in ("COPYING", "LICENSE"):
for p in glob.glob(pat + "*"):
if os.path.isfile(p):
yield p
if self.meta.readme and os.path.isfile(self.meta.readme):
yield self.meta.readme
if self.meta.filepath.exists():
yield self.meta.filepath.name
def find_files_to_add(self, for_sdist: bool = False) -> List[Path]:
"""Traverse the project path and return a list of file names
that should be included in a sdist distribution.
If for_sdist is True, will include files like LICENSE, README and pyproject
Produce a paths list relative to the source dir.
"""
return sorted({Path(p) for p in self._find_files_iter(for_sdist)})
def format_setup_py(self) -> str:
before, extra, after = [], [], []
meta = self.meta
kwargs = {
"name": meta.name,
"version": meta.version,
"author": meta.author,
"license": meta.license_type,
"author_email": meta.author_email,
"maintainer": meta.maintainer,
"maintainer_email": meta.maintainer_email,
"description": meta.description,
"url": (meta.project_urls or {}).get("homepage", ""),
}
if meta.build:
# The build script must contain a `build(setup_kwargs)`, we just import
# and execute it.
after.extend([
"from {} import build\n".format(meta.build.split(".")[0]),
"build(setup_kwargs)\n",
])
package_paths = meta.convert_package_paths()
if package_paths["packages"]:
extra.append(" 'packages': {},\n".format(
_format_list(package_paths["packages"], 8)))
if package_paths["package_dir"]:
extra.append(" 'package_dir': {!r},\n".format(
package_paths["package_dir"]))
if package_paths["package_data"]:
extra.append(" 'package_data': {!r},\n".format(
package_paths["package_data"]))
if package_paths["exclude_package_data"]:
extra.append(" 'exclude_package_data': {!r},\n".format(
package_paths["exclude_package_data"]))
if meta.readme:
before.append(OPEN_README.format(readme=meta.readme))
elif meta.long_description:
before.append("long_description = '''{}'''\n".format(
repr(meta.long_description)[1:-1]))
else:
before.append("long_description = None\n")
if meta.long_description_content_type:
extra.append(" 'long_description_content_type': {!r},\n".format(
meta.long_description_content_type))
if meta.keywords:
extra.append(f" 'keywords': {meta.keywords!r},\n")
if meta.classifiers:
extra.append(
f" 'classifiers': {_format_list(meta.classifiers, 8)},\n")
if meta.dependencies:
before.append(
f"INSTALL_REQUIRES = {_format_list(meta.dependencies)}\n")
extra.append(" 'install_requires': INSTALL_REQUIRES,\n")
if meta.optional_dependencies:
before.append("EXTRAS_REQUIRE = {}\n".format(
_format_dict_list(meta.optional_dependencies)))
extra.append(" 'extras_require': EXTRAS_REQUIRE,\n")
if meta.requires_python:
extra.append(f" 'python_requires': {meta.requires_python!r},\n")
if meta.entry_points:
before.append(
f"ENTRY_POINTS = {_format_dict_list(meta.entry_points)}\n")
extra.append(" 'entry_points': ENTRY_POINTS,\n")
return SETUP_FORMAT.format(before="".join(before),
after="".join(after),
extra="".join(extra),
**kwargs)
def format_pkginfo(self, full: bool = True) -> str:
meta = self.meta
content = METADATA_BASE.format(
name=meta.name or "UNKNOWN",
version=meta.version or "UNKNOWN",
license=meta.license_type or "UNKNOWN",
description=meta.description or "UNKNOWN",
)
# Optional fields
if meta.keywords:
content += "Keywords: {}\n".format(",".join(meta.keywords))
if meta.author:
content += f"Author: {meta.author}\n"
if meta.author_email:
content += f"Author-email: {meta.author_email}\n"
if meta.maintainer:
content += f"Maintainer: {meta.maintainer}\n"
if meta.maintainer_email:
content += f"Maintainer-email: {meta.maintainer_email}\n"
if meta.requires_python:
content += f"Requires-Python: {meta.requires_python}\n"
for classifier in meta.classifiers or []:
content += f"Classifier: {classifier}\n"
if full:
for dep in sorted(meta.dependencies):
content += f"Requires-Dist: {dep}\n"
for extra, reqs in sorted(meta.requires_extra.items()):
content += f"Provides-Extra: {extra}\n"
if full:
for dep in reqs:
content += f"Requires-Dist: {dep}\n"
for url in sorted(meta.project_urls or {}):
content += f"Project-URL: {url}, {meta.project_urls[url]}\n"
if meta.long_description_content_type:
content += "Description-Content-Type: {}\n".format(
meta.long_description_content_type)
if meta.long_description:
readme = meta.long_description
if full:
content += "\n" + readme + "\n"
else:
content += "Description: {}\n".format(
textwrap.indent(readme, " " * 8,
lambda line: True).lstrip())
return content
def ensure_setup_py(self, clean: bool = True) -> Path:
"""Ensures the requirement has a setup.py ready."""
# XXX: Currently only handle PDM project, and do nothing if not.
setup_py_path = self.location.joinpath("setup.py")
if setup_py_path.is_file():
return setup_py_path
setup_py_path.write_text(self.format_setup_py(), encoding="utf-8")
# Clean this temp file when process exits
def cleanup() -> None:
try:
setup_py_path.unlink()
except OSError:
pass
if clean:
atexit.register(cleanup)
return setup_py_path
def test_convert_legacy_project():
metadata = Metadata(FIXTURES / "projects/demo-legacy/pyproject.toml")
assert metadata.version == "0.1.0"
assert metadata.dependencies == ["flask"]
assert metadata.author == ""
assert metadata.author_email == "frostming <*****@*****.**>"