def build(self, target_dir=None): # type: (Path) -> Path
self._io.writeln(' - Building <info>sdist</info>')
if target_dir is None:
target_dir = self._path / 'dist'
if not target_dir.exists():
target_dir.mkdir(parents=True)
target = target_dir / '{}-{}.tar.gz'.format(
self._package.pretty_name, self._package.version
)
gz = GzipFile(target.as_posix(), mode='wb')
tar = tarfile.TarFile(target.as_posix(), mode='w', fileobj=gz,
format=tarfile.PAX_FORMAT)
try:
tar_dir = '{}-{}'.format(
self._package.pretty_name, self._package.version
)
files_to_add = self.find_files_to_add(exclude_build=False)
for relpath in files_to_add:
path = self._path / relpath
tar_info = tar.gettarinfo(
str(path),
arcname=pjoin(tar_dir, str(relpath))
)
tar_info = self.clean_tarinfo(tar_info)
if tar_info.isreg():
with path.open('rb') as f:
tar.addfile(tar_info, f)
else:
tar.addfile(tar_info) # Symlinks & ?
setup = self.build_setup()
tar_info = tarfile.TarInfo(pjoin(tar_dir, 'setup.py'))
tar_info.size = len(setup)
tar.addfile(tar_info, BytesIO(setup))
pkg_info = encode(PKG_INFO.format(
name=self._meta.name,
version=self._meta.version,
summary=self._meta.summary,
home_page=self._meta.home_page,
author=to_str(self._meta.author),
author_email=to_str(self._meta.author_email),
))
tar_info = tarfile.TarInfo(pjoin(tar_dir, 'PKG-INFO'))
tar_info.size = len(pkg_info)
tar.addfile(tar_info, BytesIO(pkg_info))
finally:
tar.close()
gz.close()
self._io.writeln(' - Built <fg=cyan>{}</>'.format(target.name))
return target
def build_pkg_info(self):
pkg_info = PKG_INFO.format(
name=self._meta.name,
version=self._meta.version,
summary=self._meta.summary,
home_page=self._meta.home_page,
author=to_str(self._meta.author),
author_email=to_str(self._meta.author_email),
)
if self._meta.keywords:
pkg_info += "Keywords: {}\n".format(self._meta.keywords)
if self._meta.requires_python:
pkg_info += "Requires-Python: {}\n".format(self._meta.requires_python)
for classifier in self._meta.classifiers:
pkg_info += "Classifier: {}\n".format(classifier)
for extra in sorted(self._meta.provides_extra):
pkg_info += "Provides-Extra: {}\n".format(extra)
for dep in sorted(self._meta.requires_dist):
pkg_info += "Requires-Dist: {}\n".format(dep)
return encode(pkg_info)
def convert_dependencies(
cls,
package, # type: Package
dependencies, # type: List[Dependency]
):
main = []
extras = defaultdict(list)
req_regex = re.compile(r"^(.+) \((.+)\)$")
for dependency in dependencies:
if dependency.is_optional():
for extra_name, reqs in package.extras.items():
for req in reqs:
if req.name == dependency.name:
requirement = to_str(
dependency.to_pep_508(with_extras=False)
)
if ";" in requirement:
requirement, conditions = requirement.split(";")
requirement = requirement.strip()
if req_regex.match(requirement):
requirement = req_regex.sub(
"\\1\\2", requirement.strip()
)
extras[extra_name + ":" + conditions.strip()].append(
requirement
)
continue
requirement = requirement.strip()
if req_regex.match(requirement):
requirement = req_regex.sub(
"\\1\\2", requirement.strip()
)
extras[extra_name].append(requirement)
continue
requirement = to_str(dependency.to_pep_508())
if ";" in requirement:
requirement, conditions = requirement.split(";")
requirement = requirement.strip()
if req_regex.match(requirement):
requirement = req_regex.sub("\\1\\2", requirement.strip())
extras[":" + conditions.strip()].append(requirement)
continue
requirement = requirement.strip()
if req_regex.match(requirement):
requirement = req_regex.sub("\\1\\2", requirement.strip())
main.append(requirement)
return main, dict(extras)
def convert_dependencies(
cls, package, dependencies # type: Package # type: List[Dependency]
):
main = []
extras = defaultdict(list)
req_regex = re.compile("^(.+) \((.+)\)$")
for dependency in dependencies:
if dependency.is_optional():
for extra_name, reqs in package.extras.items():
for req in reqs:
if req.name == dependency.name:
requirement = to_str(
dependency.to_pep_508(with_extras=False)
)
if ";" in requirement:
requirement, conditions = requirement.split(";")
requirement = requirement.strip()
if req_regex.match(requirement):
requirement = req_regex.sub(
"\\1\\2", requirement.strip()
)
extras[extra_name + ":" + conditions.strip()].append(
requirement
)
continue
requirement = requirement.strip()
if req_regex.match(requirement):
requirement = req_regex.sub(
"\\1\\2", requirement.strip()
)
extras[extra_name].append(requirement)
continue
requirement = to_str(dependency.to_pep_508())
if ";" in requirement:
requirement, conditions = requirement.split(";")
requirement = requirement.strip()
if req_regex.match(requirement):
requirement = req_regex.sub("\\1\\2", requirement.strip())
extras[":" + conditions.strip()].append(requirement)
continue
requirement = requirement.strip()
if req_regex.match(requirement):
requirement = req_regex.sub("\\1\\2", requirement.strip())
main.append(requirement)
return main, dict(extras)
def build_setup(self): # type: () -> bytes
before, extra, after = [], [], []
# If we have a build script, use it
if self._package.build:
after += [
"from {} import *".format(self._package.build.split(".")[0]),
"build(setup_kwargs)",
]
if self._module.is_in_src():
before.append("package_dir = \\\n{}\n".format(pformat({"": "src"})))
extra.append("'package_dir': package_dir,")
if self._module.is_package():
packages, package_data = self.find_packages(self._module.path.as_posix())
before.append("packages = \\\n{}\n".format(pformat(sorted(packages))))
before.append("package_data = \\\n{}\n".format(pformat(package_data)))
extra.append("'packages': packages,")
extra.append("'package_data': package_data,")
else:
extra.append("'py_modules': {!r},".format(to_str(self._module.name)))
dependencies, extras = self.convert_dependencies(
self._package, self._package.requires
)
if dependencies:
before.append(
"install_requires = \\\n{}\n".format(pformat(sorted(dependencies)))
)
extra.append("'install_requires': install_requires,")
if extras:
before.append("extras_require = \\\n{}\n".format(pformat(extras)))
extra.append("'extras_require': extras_require,")
entry_points = self.convert_entry_points()
if entry_points:
before.append("entry_points = \\\n{}\n".format(pformat(entry_points)))
extra.append("'entry_points': entry_points,")
if self._package.python_versions != "*":
python_requires = self._meta.requires_python
extra.append("'python_requires': {!r},".format(python_requires))
return encode(
SETUP.format(
before="\n".join(before),
name=to_str(self._meta.name),
version=to_str(self._meta.version),
description=to_str(self._meta.summary),
long_description=to_str(self._meta.description),
author=to_str(self._meta.author),
author_email=to_str(self._meta.author_email),
url=to_str(self._meta.home_page),
extra="\n ".join(extra),
after="\n".join(after),
)
)
def search(self, query, mode=0):
results = []
search = {"name": query}
if mode == self.SEARCH_FULLTEXT:
search["summary"] = query
client = ServerProxy("https://pypi.python.org/pypi")
hits = client.search(search, "or")
for hit in hits:
try:
result = Package(hit["name"], hit["version"], hit["version"])
result.description = to_str(hit["summary"])
results.append(result)
except ParseVersionError:
self._log(
'Unable to parse version "{}" for the {} package, skipping'.format(
hit["version"], hit["name"]
),
level="debug",
)
return results
def search(self, query: str) -> List[Package]:
results = []
search = {"q": query}
response = requests.session().get(self._base_url + "search",
params=search)
content = parse(response.content, namespaceHTMLElements=False)
for result in content.findall(".//*[@class='package-snippet']"):
name = result.find("h3/*[@class='package-snippet__name']").text
version = result.find(
"h3/*[@class='package-snippet__version']").text
if not name or not version:
continue
description = result.find(
"p[@class='package-snippet__description']").text
if not description:
description = ""
try:
result = Package(name, version, description)
result.description = to_str(description.strip())
results.append(result)
except ParseVersionError:
self._log(
'Unable to parse version "{}" for the {} package, skipping'
.format(version, name),
level="debug",
)
return results
def test_with_src_module_file():
poetry = Poetry.create(project('source_file'))
builder = SdistBuilder(poetry, NullVenv(), NullIO())
# Check setup.py
setup = builder.build_setup()
setup_ast = ast.parse(setup)
setup_ast.body = [n for n in setup_ast.body if isinstance(n, ast.Assign)]
ns = {}
exec(compile(setup_ast, filename="setup.py", mode="exec"), ns)
assert ns['package_dir'] == {'': 'src'}
assert re.search('\'py_modules\': \'module_src\'',
to_str(setup)) is not None
builder.build()
sdist = fixtures_dir / 'source_file' / 'dist' / 'module-src-0.1.tar.gz'
assert sdist.exists()
tar = tarfile.open(str(sdist), 'r')
assert 'module-src-0.1/src/module_src.py' in tar.getnames()
def get_metadata_content(self): # type: () -> bytes
content = METADATA_BASE.format(
name=self._meta.name,
version=self._meta.version,
summary=to_str(self._meta.summary),
)
# Optional fields
if self._meta.home_page:
content += "Home-page: {}\n".format(self._meta.home_page)
if self._meta.license:
content += "License: {}\n".format(self._meta.license)
if self._meta.keywords:
content += "Keywords: {}\n".format(self._meta.keywords)
if self._meta.author:
content += "Author: {}\n".format(to_str(self._meta.author))
if self._meta.author_email:
content += "Author-email: {}\n".format(to_str(self._meta.author_email))
if self._meta.requires_python:
content += "Requires-Python: {}\n".format(self._meta.requires_python)
for classifier in self._meta.classifiers:
content += "Classifier: {}\n".format(classifier)
for extra in sorted(self._meta.provides_extra):
content += "Provides-Extra: {}\n".format(extra)
for dep in sorted(self._meta.requires_dist):
content += "Requires-Dist: {}\n".format(dep)
for url in sorted(self._meta.project_urls, key=lambda u: u[0]):
content += "Project-URL: {}\n".format(to_str(url))
if self._meta.description_content_type:
content += "Description-Content-Type: {}\n".format(
self._meta.description_content_type
)
if self._meta.description is not None:
content += "\n" + to_str(self._meta.description) + "\n"
return content
def test_proper_python_requires_if_three_digits_precision_version_specified():
poetry = Poetry.create(project("single_python"))
builder = SdistBuilder(poetry, NullEnv(), NullIO())
pkg_info = builder.build_pkg_info()
p = Parser()
parsed = p.parsestr(to_str(pkg_info))
assert parsed["Requires-Python"] == "==2.7.15"
def test_make_pkg_info_any_python():
poetry = Poetry.create(project("module1"))
builder = SdistBuilder(poetry, NullEnv(), NullIO())
pkg_info = builder.build_pkg_info()
p = Parser()
parsed = p.parsestr(to_str(pkg_info))
assert "Requires-Python" not in parsed
def test_proper_python_requires_if_two_digits_precision_version_specified():
poetry = Factory().create_poetry(project("simple_version"))
builder = SdistBuilder(poetry, NullEnv(), NullIO())
pkg_info = builder.build_pkg_info()
p = Parser()
parsed = p.parsestr(to_str(pkg_info))
assert parsed["Requires-Python"] == ">=3.6,<3.7"
def test_proper_python_requires_if_single_version_specified():
poetry = Poetry.create(project("simple_version"))
builder = SdistBuilder(poetry, NullVenv(), NullIO())
pkg_info = builder.build_pkg_info()
p = Parser()
parsed = p.parsestr(to_str(pkg_info))
assert parsed["Requires-Python"] == ">=3.6,<3.7"
def test_make_pkg_info_multi_constraints_dependency():
poetry = Poetry.create(
Path(__file__).parent.parent.parent / "fixtures" /
"project_with_multi_constraints_dependency")
builder = SdistBuilder(poetry, NullEnv(), NullIO())
pkg_info = builder.build_pkg_info()
p = Parser()
parsed = p.parsestr(to_str(pkg_info))
requires = parsed.get_all("Requires-Dist")
assert requires == [
'pendulum (>=1.5,<2.0); python_version < "3.4"',
'pendulum (>=2.0,<3.0); python_version >= "3.4" and python_version < "4.0"',
]
def search(self, query, mode=0):
results = []
search = {"name": query}
if mode == self.SEARCH_FULLTEXT:
search["summary"] = query
client = ServerProxy("https://pypi.python.org/pypi")
hits = client.search(search, "or")
for hit in hits:
result = Package(hit["name"], hit["version"], hit["version"])
result.description = to_str(hit["summary"])
results.append(result)
return results
def search(self, query, mode=0):
results = []
search = {'name': query}
if mode == self.SEARCH_FULLTEXT:
search['summary'] = query
client = ServerProxy('https://pypi.python.org/pypi')
hits = client.search(search, 'or')
for hit in hits:
result = Package(hit['name'], hit['version'], hit['version'])
result.description = to_str(hit['summary'])
results.append(result)
return results
def search(self, query, mode=0):
results = []
search = {"name": query}
if mode == self.SEARCH_FULLTEXT:
search["summary"] = query
client = ServerProxy("https://pypi.python.org/pypi")
hits = client.search(search, "or")
for hit in hits:
result = Package(hit["name"], hit["version"], hit["version"])
result.description = to_str(hit["summary"])
results.append(result)
return results
def test_metadata_file_with_vcs_dependencies():
project_path = fixtures_dir / "with_vcs_dependency"
WheelBuilder.make(Poetry.create(str(project_path)), NullEnv(), NullIO())
whl = project_path / "dist" / "with_vcs_dependency-1.2.3-py3-none-any.whl"
assert whl.exists()
p = Parser()
with zipfile.ZipFile(str(whl)) as z:
metadata = p.parsestr(
to_str(z.read("with_vcs_dependency-1.2.3.dist-info/METADATA"))
)
requires_dist = metadata["Requires-Dist"]
assert "cleo @ git+https://github.com/sdispater/cleo.git@master" == requires_dist
def search(self, query: str) -> list[Package]:
results = []
search = {"q": query}
response = requests.session().get(self._base_url + "search", params=search)
content = parse(response.content, namespaceHTMLElements=False)
for result in content.findall(".//*[@class='package-snippet']"):
name_element = result.find("h3/*[@class='package-snippet__name']")
version_element = result.find("h3/*[@class='package-snippet__version']")
if (
name_element is None
or version_element is None
or not name_element.text
or not version_element.text
):
continue
name = name_element.text
version = version_element.text
description_element = result.find(
"p[@class='package-snippet__description']"
)
description = (
description_element.text
if description_element is not None and description_element.text
else ""
)
try:
package = Package(name, version)
package.description = to_str(description.strip())
results.append(package)
except InvalidVersion:
self._log(
f'Unable to parse version "{version}" for the {name} package,'
" skipping",
level="debug",
)
return results
def test_make_pkg_info():
poetry = Poetry.create(project("complete"))
builder = SdistBuilder(poetry, NullEnv(), NullIO())
pkg_info = builder.build_pkg_info()
p = Parser()
parsed = p.parsestr(to_str(pkg_info))
assert parsed["Metadata-Version"] == "2.1"
assert parsed["Name"] == "my-package"
assert parsed["Version"] == "1.2.3"
assert parsed["Summary"] == "Some description."
assert parsed["Author"] == "Sébastien Eustace"
assert parsed["Author-email"] == "*****@*****.**"
assert parsed["Keywords"] == "packaging,dependency,poetry"
assert parsed["Requires-Python"] == ">=3.6,<4.0"
classifiers = parsed.get_all("Classifier")
assert classifiers == [
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Topic :: Software Development :: Build Tools",
"Topic :: Software Development :: Libraries :: Python Modules",
]
extras = parsed.get_all("Provides-Extra")
assert extras == ["time"]
requires = parsed.get_all("Requires-Dist")
assert requires == [
"cachy[msgpack] (>=0.2.0,<0.3.0)",
"cleo (>=0.6,<0.7)",
'pendulum (>=1.4,<2.0); extra == "time"',
]
urls = parsed.get_all("Project-URL")
assert urls == [
"Documentation, https://poetry.eustace.io/docs",
"Repository, https://github.com/sdispater/poetry",
]
def test_with_src_module_file():
poetry = Poetry.create(project("source_file"))
builder = SdistBuilder(poetry, NullVenv(), NullIO())
# Check setup.py
setup = builder.build_setup()
setup_ast = ast.parse(setup)
setup_ast.body = [n for n in setup_ast.body if isinstance(n, ast.Assign)]
ns = {}
exec(compile(setup_ast, filename="setup.py", mode="exec"), ns)
assert ns["package_dir"] == {"": "src"}
assert re.search("'py_modules': 'module_src'", to_str(setup)) is not None
builder.build()
sdist = fixtures_dir / "source_file" / "dist" / "module-src-0.1.tar.gz"
assert sdist.exists()
tar = tarfile.open(str(sdist), "r")
assert "module-src-0.1/src/module_src.py" in tar.getnames()
def build_setup(self): # type: () -> bytes
before, extra, after = [], [], []
package_dir = {}
# If we have a build script, use it
if self._package.build:
after += [
"from {} import *".format(self._package.build.split(".")[0]),
"build(setup_kwargs)",
]
modules = []
packages = []
package_data = {}
for include in self._module.includes:
if isinstance(include, PackageInclude):
if include.is_package():
pkg_dir, _packages, _package_data = self.find_packages(include)
if pkg_dir is not None:
package_dir[""] = os.path.relpath(pkg_dir, str(self._path))
packages += _packages
package_data.update(_package_data)
else:
if include.source is not None:
package_dir[""] = str(include.base.relative_to(self._path))
modules.append(include.elements[0].relative_to(include.base).stem)
else:
pass
if package_dir:
before.append("package_dir = \\\n{}\n".format(pformat(package_dir)))
extra.append("'package_dir': package_dir,")
if packages:
before.append("packages = \\\n{}\n".format(pformat(sorted(packages))))
extra.append("'packages': packages,")
if package_data:
before.append("package_data = \\\n{}\n".format(pformat(package_data)))
extra.append("'package_data': package_data,")
if modules:
before.append("modules = \\\n{}".format(pformat(modules)))
extra.append("'py_modules': modules,".format())
dependencies, extras = self.convert_dependencies(
self._package, self._package.requires
)
if dependencies:
before.append(
"install_requires = \\\n{}\n".format(pformat(sorted(dependencies)))
)
extra.append("'install_requires': install_requires,")
if extras:
before.append("extras_require = \\\n{}\n".format(pformat(extras)))
extra.append("'extras_require': extras_require,")
entry_points = self.convert_entry_points()
if entry_points:
before.append("entry_points = \\\n{}\n".format(pformat(entry_points)))
extra.append("'entry_points': entry_points,")
if self._package.python_versions != "*":
python_requires = self._meta.requires_python
extra.append("'python_requires': {!r},".format(python_requires))
return encode(
SETUP.format(
before="\n".join(before),
name=to_str(self._meta.name),
version=to_str(self._meta.version),
description=to_str(self._meta.summary),
long_description=to_str(self._meta.description),
author=to_str(self._meta.author),
author_email=to_str(self._meta.author_email),
url=to_str(self._meta.home_page),
extra="\n ".join(extra),
after="\n".join(after),
)
)