def _inspect_sdist_with_setup(self, sdist_dir):
info = {"requires_python": None, "requires_dist": None}
setup = sdist_dir / "setup.py"
if not setup.exists():
return info
venv = Venv.create(NullIO())
current_dir = os.getcwd()
os.chdir(sdist_dir.as_posix())
try:
venv.run("python", "setup.py", "egg_info")
egg_info = list(sdist_dir.glob("**/*.egg-info"))[0]
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
else:
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
except Exception:
pass
os.chdir(current_dir)
return info
def _inspect_sdist_with_setup(self, sdist_dir):
info = {"requires_python": None, "requires_dist": None}
setup = sdist_dir / "setup.py"
if not setup.exists():
return info
venv = Venv.create(NullIO())
current_dir = os.getcwd()
os.chdir(sdist_dir.as_posix())
try:
venv.run("python", "setup.py", "egg_info")
egg_info = list(sdist_dir.glob("**/*.egg-info"))[0]
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
else:
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
except Exception:
pass
os.chdir(current_dir)
return info
def test_parse_requires():
requires = """\
jsonschema>=2.6.0.0,<3.0.0.0
lockfile>=0.12.0.0,<0.13.0.0
pip-tools>=1.11.0.0,<2.0.0.0
pkginfo>=1.4.0.0,<2.0.0.0
pyrsistent>=0.14.2.0,<0.15.0.0
toml>=0.9.0.0,<0.10.0.0
cleo>=0.6.0.0,<0.7.0.0
cachy>=0.1.1.0,<0.2.0.0
cachecontrol>=0.12.4.0,<0.13.0.0
requests>=2.18.0.0,<3.0.0.0
msgpack-python>=0.5.0.0,<0.6.0.0
pyparsing>=2.2.0.0,<3.0.0.0
requests-toolbelt>=0.8.0.0,<0.9.0.0
[:(python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")]
typing>=3.6.0.0,<4.0.0.0
[:python_version >= "2.7.0.0" and python_version < "2.8.0.0"]
virtualenv>=15.2.0.0,<16.0.0.0
pathlib2>=2.3.0.0,<3.0.0.0
[:python_version >= "3.4.0.0" and python_version < "3.6.0.0"]
zipfile36>=0.1.0.0,<0.2.0.0
[dev]
isort@ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
"""
result = parse_requires(requires)
expected = [
"jsonschema>=2.6.0.0,<3.0.0.0",
"lockfile>=0.12.0.0,<0.13.0.0",
"pip-tools>=1.11.0.0,<2.0.0.0",
"pkginfo>=1.4.0.0,<2.0.0.0",
"pyrsistent>=0.14.2.0,<0.15.0.0",
"toml>=0.9.0.0,<0.10.0.0",
"cleo>=0.6.0.0,<0.7.0.0",
"cachy>=0.1.1.0,<0.2.0.0",
"cachecontrol>=0.12.4.0,<0.13.0.0",
"requests>=2.18.0.0,<3.0.0.0",
"msgpack-python>=0.5.0.0,<0.6.0.0",
"pyparsing>=2.2.0.0,<3.0.0.0",
"requests-toolbelt>=0.8.0.0,<0.9.0.0",
'typing>=3.6.0.0,<4.0.0.0 ; (python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")',
'virtualenv>=15.2.0.0,<16.0.0.0 ; python_version >= "2.7.0.0" and python_version < "2.8.0.0"',
'pathlib2>=2.3.0.0,<3.0.0.0 ; python_version >= "2.7.0.0" and python_version < "2.8.0.0"',
'zipfile36>=0.1.0.0,<0.2.0.0 ; python_version >= "3.4.0.0" and python_version < "3.6.0.0"',
'isort@ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort ; extra == "dev"',
]
assert result == expected
def test_parse_requires():
requires = """\
jsonschema>=2.6.0.0,<3.0.0.0
lockfile>=0.12.0.0,<0.13.0.0
pip-tools>=1.11.0.0,<2.0.0.0
pkginfo>=1.4.0.0,<2.0.0.0
pyrsistent>=0.14.2.0,<0.15.0.0
toml>=0.9.0.0,<0.10.0.0
cleo>=0.6.0.0,<0.7.0.0
cachy>=0.1.1.0,<0.2.0.0
cachecontrol>=0.12.4.0,<0.13.0.0
requests>=2.18.0.0,<3.0.0.0
msgpack-python>=0.5.0.0,<0.6.0.0
pyparsing>=2.2.0.0,<3.0.0.0
requests-toolbelt>=0.8.0.0,<0.9.0.0
[:(python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")]
typing>=3.6.0.0,<4.0.0.0
[:python_version >= "2.7.0.0" and python_version < "2.8.0.0"]
virtualenv>=15.2.0.0,<16.0.0.0
pathlib2>=2.3.0.0,<3.0.0.0
[:python_version >= "3.4.0.0" and python_version < "3.6.0.0"]
zipfile36>=0.1.0.0,<0.2.0.0
"""
result = parse_requires(requires)
expected = [
"jsonschema>=2.6.0.0,<3.0.0.0",
"lockfile>=0.12.0.0,<0.13.0.0",
"pip-tools>=1.11.0.0,<2.0.0.0",
"pkginfo>=1.4.0.0,<2.0.0.0",
"pyrsistent>=0.14.2.0,<0.15.0.0",
"toml>=0.9.0.0,<0.10.0.0",
"cleo>=0.6.0.0,<0.7.0.0",
"cachy>=0.1.1.0,<0.2.0.0",
"cachecontrol>=0.12.4.0,<0.13.0.0",
"requests>=2.18.0.0,<3.0.0.0",
"msgpack-python>=0.5.0.0,<0.6.0.0",
"pyparsing>=2.2.0.0,<3.0.0.0",
"requests-toolbelt>=0.8.0.0,<0.9.0.0",
'typing>=3.6.0.0,<4.0.0.0; (python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")',
'virtualenv>=15.2.0.0,<16.0.0.0; python_version >= "2.7.0.0" and python_version < "2.8.0.0"',
'pathlib2>=2.3.0.0,<3.0.0.0; python_version >= "2.7.0.0" and python_version < "2.8.0.0"',
'zipfile36>=0.1.0.0,<0.2.0.0; python_version >= "3.4.0.0" and python_version < "3.6.0.0"',
]
assert result == expected
def test_parse_requires():
requires = """\
jsonschema>=2.6.0.0,<3.0.0.0
lockfile>=0.12.0.0,<0.13.0.0
pip-tools>=1.11.0.0,<2.0.0.0
pkginfo>=1.4.0.0,<2.0.0.0
pyrsistent>=0.14.2.0,<0.15.0.0
toml>=0.9.0.0,<0.10.0.0
cleo>=0.6.0.0,<0.7.0.0
cachy>=0.1.1.0,<0.2.0.0
cachecontrol>=0.12.4.0,<0.13.0.0
requests>=2.18.0.0,<3.0.0.0
msgpack-python>=0.5.0.0,<0.6.0.0
pyparsing>=2.2.0.0,<3.0.0.0
requests-toolbelt>=0.8.0.0,<0.9.0.0
[:(python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")]
typing>=3.6.0.0,<4.0.0.0
[:python_version >= "2.7.0.0" and python_version < "2.8.0.0"]
virtualenv>=15.2.0.0,<16.0.0.0
pathlib2>=2.3.0.0,<3.0.0.0
[:python_version >= "3.4.0.0" and python_version < "3.6.0.0"]
zipfile36>=0.1.0.0,<0.2.0.0
"""
result = parse_requires(requires)
expected = [
"jsonschema>=2.6.0.0,<3.0.0.0",
"lockfile>=0.12.0.0,<0.13.0.0",
"pip-tools>=1.11.0.0,<2.0.0.0",
"pkginfo>=1.4.0.0,<2.0.0.0",
"pyrsistent>=0.14.2.0,<0.15.0.0",
"toml>=0.9.0.0,<0.10.0.0",
"cleo>=0.6.0.0,<0.7.0.0",
"cachy>=0.1.1.0,<0.2.0.0",
"cachecontrol>=0.12.4.0,<0.13.0.0",
"requests>=2.18.0.0,<3.0.0.0",
"msgpack-python>=0.5.0.0,<0.6.0.0",
"pyparsing>=2.2.0.0,<3.0.0.0",
"requests-toolbelt>=0.8.0.0,<0.9.0.0",
'typing>=3.6.0.0,<4.0.0.0; (python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")',
'virtualenv>=15.2.0.0,<16.0.0.0; python_version >= "2.7.0.0" and python_version < "2.8.0.0"',
'pathlib2>=2.3.0.0,<3.0.0.0; python_version >= "2.7.0.0" and python_version < "2.8.0.0"',
'zipfile36>=0.1.0.0,<0.2.0.0; python_version >= "3.4.0.0" and python_version < "3.6.0.0"',
]
assert result == expected
def _inspect_sdist_with_setup(self, sdist_dir):
info = {"requires_python": None, "requires_dist": None}
result = SetupReader.read_from_directory(sdist_dir)
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
info["requires_dist"] = parse_requires(requires)
info["requires_python"] = result["python_requires"]
return info
def _inspect_sdist_with_setup(self, sdist_dir):
info = {
'requires_python': None,
'requires_dist': None,
}
setup = sdist_dir / 'setup.py'
if not setup.exists():
return info
venv = Venv.create(NullIO())
current_dir = os.getcwd()
os.chdir(sdist_dir.as_posix())
try:
venv.run('python', 'setup.py', 'egg_info')
egg_info = list(sdist_dir.glob('**/*.egg-info'))[0]
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_python:
info['requires_python'] = meta.requires_python
if meta.requires_dist:
info['requires_dist'] = list(meta.requires_dist)
else:
requires = egg_info / 'requires.txt'
if requires.exists():
with requires.open() as f:
info['requires_dist'] = parse_requires(f.read())
except Exception:
pass
os.chdir(current_dir)
return info
def search_for_vcs(self,
dependency): # type: (VCSDependency) -> List[Package]
"""
Search for the specifications that match the given VCS dependency.
Basically, we clone the repository in a temporary directory
and get the information we need by checking out the specified reference.
"""
if dependency.vcs != "git":
raise ValueError("Unsupported VCS dependency {}".format(
dependency.vcs))
tmp_dir = Path(
mkdtemp(prefix="pypoetry-git-{}".format(dependency.name)))
try:
git = Git()
git.clone(dependency.source, tmp_dir)
git.checkout(dependency.reference, tmp_dir)
revision = git.rev_parse(dependency.reference, tmp_dir).strip()
if dependency.tag or dependency.rev:
revision = dependency.reference
pyproject = TomlFile(tmp_dir / "pyproject.toml")
pyproject_content = None
has_poetry = False
if pyproject.exists():
pyproject_content = pyproject.read()
has_poetry = ("tool" in pyproject_content
and "poetry" in pyproject_content["tool"])
if pyproject_content and has_poetry:
# If a pyproject.toml file exists
# We use it to get the information we need
info = pyproject_content["tool"]["poetry"]
name = info["name"]
version = info["version"]
package = Package(name, version, version)
package.source_type = dependency.vcs
package.source_url = dependency.source
package.source_reference = dependency.reference
for req_name, req_constraint in info["dependencies"].items():
if req_name == "python":
package.python_versions = req_constraint
continue
package.add_dependency(req_name, req_constraint)
else:
# We need to use setup.py here
# to figure the information we need
# We need to place ourselves in the proper
# folder for it to work
venv = Venv.create(self._io)
current_dir = os.getcwd()
os.chdir(tmp_dir.as_posix())
try:
venv.run("python", "setup.py", "egg_info")
# Sometimes pathlib will fail on recursive
# symbolic links, so we need to workaround it
# and use the glob module instead.
# Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
if PY35:
egg_info = next(
Path(p) for p in glob.glob(
os.path.join(str(tmp_dir), "**", "*.egg-info"),
recursive=True,
))
else:
egg_info = next(tmp_dir.glob("**/*.egg-info"))
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
reqs = parse_requires(f.read())
package = Package(meta.name, meta.version)
for req in reqs:
dep = dependency_from_pep_508(req)
if dep.in_extras:
for extra in dep.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dep)
package.requires.append(dep)
except Exception:
raise
finally:
os.chdir(current_dir)
package.source_type = "git"
package.source_url = dependency.source
package.source_reference = revision
except Exception:
raise
finally:
shutil.rmtree(tmp_dir.as_posix())
if dependency.name != package.name:
# For now, the dependency's name must match the actual package's name
raise RuntimeError(
"The dependency name for {} does not match the actual package's name: {}"
.format(dependency.name, package.name))
if dependency.extras:
for extra in dependency.extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
return [package]
def search_for_vcs(self,
dependency): # type: (VCSDependency) -> List[Package]
"""
Search for the specifications that match the given VCS dependency.
Basically, we clone the repository in a temporary directory
and get the information we need by checking out the specified reference.
"""
if dependency.vcs != 'git':
raise ValueError('Unsupported VCS dependency {}'.format(
dependency.vcs))
tmp_dir = Path(
mkdtemp(prefix='pypoetry-git-{}'.format(dependency.name)))
try:
git = Git()
git.clone(dependency.source, tmp_dir)
git.checkout(dependency.reference, tmp_dir)
revision = git.rev_parse(dependency.reference, tmp_dir).strip()
if dependency.tag or dependency.rev:
revision = dependency.reference
pyproject = TomlFile(tmp_dir / 'pyproject.toml')
pyproject_content = None
has_poetry = False
if pyproject.exists():
pyproject_content = pyproject.read(True)
has_poetry = ('tool' in pyproject_content
and 'poetry' in pyproject_content['tool'])
if pyproject_content and has_poetry:
# If a pyproject.toml file exists
# We use it to get the information we need
info = pyproject_content['tool']['poetry']
name = info['name']
version = info['version']
package = Package(name, version, version)
package.source_type = dependency.vcs
package.source_url = dependency.source
package.source_reference = dependency.reference
for req_name, req_constraint in info['dependencies'].items():
if req_name == 'python':
package.python_versions = req_constraint
continue
package.add_dependency(req_name, req_constraint)
else:
# We need to use setup.py here
# to figure the information we need
# We need to place ourselves in the proper
# folder for it to work
venv = Venv.create(self._io)
current_dir = os.getcwd()
os.chdir(tmp_dir.as_posix())
try:
venv.run('python', 'setup.py', 'egg_info')
egg_info = list(tmp_dir.glob('*.egg-info'))[0]
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / 'requires.txt'
if requires.exists():
with requires.open() as f:
reqs = parse_requires(f.read())
package = Package(meta.name, meta.version)
for req in reqs:
package.requires.append(dependency_from_pep_508(req))
except Exception:
raise
finally:
os.chdir(current_dir)
package.source_type = 'git'
package.source_url = dependency.source
package.source_reference = revision
except Exception:
raise
finally:
shutil.rmtree(tmp_dir.as_posix())
return [package]
def _get_info_from_sdist(
self, url
): # type: (str) -> Dict[str, Union[str, List, None]]
info = {"summary": "", "requires_python": None, "requires_dist": None}
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
try:
meta = pkginfo.SDist(str(filepath))
if meta.summary:
info["summary"] = meta.summary
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
return info
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = filepath.suffix
gz = None
if suffix == ".zip":
tar = zipfile.ZipFile(str(filepath))
else:
if suffix == ".bz2":
gz = BZ2File(str(filepath))
else:
gz = GzipFile(str(filepath))
tar = tarfile.TarFile(str(filepath), fileobj=gz)
try:
tar.extractall(os.path.join(temp_dir, "unpacked"))
finally:
if gz:
gz.close()
tar.close()
unpacked = Path(temp_dir) / "unpacked"
sdist_dir = unpacked / Path(filename).name.rstrip(".tar.gz")
# Checking for .egg-info at root
eggs = list(sdist_dir.glob("*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Searching for .egg-info in sub directories
eggs = list(sdist_dir.glob("**/*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Still nothing, assume no dependencies
# We could probably get them by executing
# python setup.py egg-info but I don't feel
# confortable executing a file just for the sake
# of getting dependencies.
return info
def _get_info_from_sdist(
self, url): # type: (str) -> Dict[str, Union[str, List, None]]
info = {"summary": "", "requires_python": None, "requires_dist": None}
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
try:
meta = pkginfo.SDist(str(filepath))
if meta.summary:
info["summary"] = meta.summary
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
return info
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = filepath.suffix
gz = None
if suffix == ".zip":
tar = zipfile.ZipFile(str(filepath))
else:
if suffix == ".bz2":
gz = BZ2File(str(filepath))
else:
gz = GzipFile(str(filepath))
suffix = ".tar.gz"
tar = tarfile.TarFile(str(filepath), fileobj=gz)
try:
tar.extractall(os.path.join(temp_dir, "unpacked"))
finally:
if gz:
gz.close()
tar.close()
unpacked = Path(temp_dir) / "unpacked"
sdist_dir = unpacked / Path(filename).name.rstrip(suffix)
# Checking for .egg-info at root
eggs = list(sdist_dir.glob("*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Searching for .egg-info in sub directories
eggs = list(sdist_dir.glob("**/*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Still nothing, assume no dependencies
# We could probably get them by executing
# python setup.py egg-info but I don't feel
# confortable executing a file just for the sake
# of getting dependencies.
return info
def search_for_vcs(self, dependency): # type: (VCSDependency) -> List[Package]
"""
Search for the specifications that match the given VCS dependency.
Basically, we clone the repository in a temporary directory
and get the information we need by checking out the specified reference.
"""
if dependency.vcs != "git":
raise ValueError("Unsupported VCS dependency {}".format(dependency.vcs))
tmp_dir = Path(mkdtemp(prefix="pypoetry-git-{}".format(dependency.name)))
try:
git = Git()
git.clone(dependency.source, tmp_dir)
git.checkout(dependency.reference, tmp_dir)
revision = git.rev_parse(dependency.reference, tmp_dir).strip()
if dependency.tag or dependency.rev:
revision = dependency.reference
pyproject = TomlFile(tmp_dir / "pyproject.toml")
pyproject_content = None
has_poetry = False
if pyproject.exists():
pyproject_content = pyproject.read()
has_poetry = (
"tool" in pyproject_content
and "poetry" in pyproject_content["tool"]
)
if pyproject_content and has_poetry:
# If a pyproject.toml file exists
# We use it to get the information we need
info = pyproject_content["tool"]["poetry"]
name = info["name"]
version = info["version"]
package = Package(name, version, version)
package.source_type = dependency.vcs
package.source_url = dependency.source
package.source_reference = dependency.reference
for req_name, req_constraint in info["dependencies"].items():
if req_name == "python":
package.python_versions = req_constraint
continue
package.add_dependency(req_name, req_constraint)
else:
# We need to use setup.py here
# to figure the information we need
# We need to place ourselves in the proper
# folder for it to work
venv = Venv.create(self._io)
current_dir = os.getcwd()
os.chdir(tmp_dir.as_posix())
try:
venv.run("python", "setup.py", "egg_info")
# Sometimes pathlib will fail on recursive
# symbolic links, so we need to workaround it
# and use the glob module instead.
# Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
if PY35:
egg_info = next(
Path(p)
for p in glob.glob(
os.path.join(str(tmp_dir), "**", "*.egg-info"),
recursive=True,
)
)
else:
egg_info = next(tmp_dir.glob("**/*.egg-info"))
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
reqs = parse_requires(f.read())
package = Package(meta.name, meta.version)
for req in reqs:
dep = dependency_from_pep_508(req)
if dep.in_extras:
for extra in dep.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dep)
package.requires.append(dep)
except Exception:
raise
finally:
os.chdir(current_dir)
package.source_type = "git"
package.source_url = dependency.source
package.source_reference = revision
except Exception:
raise
finally:
shutil.rmtree(tmp_dir.as_posix())
if dependency.name != package.name:
# For now, the dependency's name must match the actual package's name
raise RuntimeError(
"The dependency name for {} does not match the actual package's name: {}".format(
dependency.name, package.name
)
)
if dependency.extras:
for extra in dependency.extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
return [package]
def __init__(
self,
path, # type: Path
category="main", # type: str
optional=False, # type: bool
base=None, # type: Path
develop=False, # type: bool
):
from . import dependency_from_pep_508
from .package import Package
self._path = path
self._base = base
self._full_path = path
self._develop = develop
if self._base and not self._path.is_absolute():
self._full_path = self._base / self._path
if not self._full_path.exists():
raise ValueError("Directory {} does not exist".format(self._path))
if self._full_path.is_file():
raise ValueError("{} is a file, expected a directory".format(
self._path))
# Checking content to dertermine actions
setup = self._full_path / "setup.py"
pyproject = TomlFile(self._full_path / "pyproject.toml")
has_poetry = False
if pyproject.exists():
pyproject_content = pyproject.read()
has_poetry = ("tool" in pyproject_content
and "poetry" in pyproject_content["tool"])
if not setup.exists() and not has_poetry:
raise ValueError(
"Directory {} does not seem to be a Python package".format(
self._full_path))
if has_poetry:
from poetry.masonry.builders import SdistBuilder
from poetry.poetry import Poetry
poetry = Poetry.create(self._full_path)
builder = SdistBuilder(poetry, NullVenv(), NullIO())
with setup.open("w") as f:
f.write(decode(builder.build_setup()))
package = poetry.package
self._package = Package(package.pretty_name, package.version)
self._package.requires += package.requires
self._package.dev_requires += package.dev_requires
self._package.extras = package.extras
self._package.python_versions = package.python_versions
self._package.platform = package.platform
else:
# Execute egg_info
current_dir = os.getcwd()
os.chdir(str(self._full_path))
try:
cwd = base
venv = Venv.create(NullIO(), cwd=cwd)
venv.run("python", "setup.py", "egg_info")
finally:
os.chdir(current_dir)
egg_info = list(self._full_path.glob("*.egg-info"))[0]
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
reqs = parse_requires(f.read())
package = Package(meta.name, meta.version)
package.description = meta.summary
for req in reqs:
package.requires.append(dependency_from_pep_508(req))
if meta.requires_python:
package.python_versions = meta.requires_python
if meta.platforms:
platforms = [
p for p in meta.platforms if p.lower() != "unknown"
]
if platforms:
package.platform = " || ".join(platforms)
self._package = package
self._package.source_type = "directory"
self._package.source_url = self._path.as_posix()
super(DirectoryDependency, self).__init__(
self._package.name,
self._package.version,
category=category,
optional=optional,
allows_prereleases=True,
)
def __init__(
self,
path, # type: Path
category="main", # type: str
optional=False, # type: bool
base=None, # type: Path
develop=False, # type: bool
):
from . import dependency_from_pep_508
from .package import Package
self._path = path
self._base = base
self._full_path = path
self._develop = develop
if self._base and not self._path.is_absolute():
self._full_path = self._base / self._path
if not self._full_path.exists():
raise ValueError("Directory {} does not exist".format(self._path))
if self._full_path.is_file():
raise ValueError("{} is a file, expected a directory".format(self._path))
# Checking content to dertermine actions
setup = self._full_path / "setup.py"
pyproject = TomlFile(self._full_path / "pyproject.toml")
has_poetry = False
if pyproject.exists():
pyproject_content = pyproject.read(True)
has_poetry = (
"tool" in pyproject_content and "poetry" in pyproject_content["tool"]
)
if not setup.exists() and not has_poetry:
raise ValueError(
"Directory {} does not seem to be a Python package".format(
self._full_path
)
)
if has_poetry:
from poetry.masonry.builders import SdistBuilder
from poetry.poetry import Poetry
poetry = Poetry.create(self._full_path)
builder = SdistBuilder(poetry, NullVenv(), NullIO())
with setup.open("w") as f:
f.write(decode(builder.build_setup()))
package = poetry.package
self._package = Package(package.pretty_name, package.version)
self._package.requires += package.requires
self._package.dev_requires += package.dev_requires
self._package.python_versions = package.python_versions
self._package.platform = package.platform
else:
# Execute egg_info
current_dir = os.getcwd()
os.chdir(str(self._full_path))
try:
cwd = base
venv = Venv.create(NullIO(), cwd=cwd)
venv.run("python", "setup.py", "egg_info")
finally:
os.chdir(current_dir)
egg_info = list(self._full_path.glob("*.egg-info"))[0]
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
reqs = parse_requires(f.read())
package = Package(meta.name, meta.version)
package.description = meta.summary
for req in reqs:
package.requires.append(dependency_from_pep_508(req))
if meta.requires_python:
package.python_versions = meta.requires_python
if meta.platforms:
platforms = [p for p in meta.platforms if p.lower() != "unknown"]
if platforms:
package.platform = " || ".join(platforms)
self._package = package
self._package.source_type = "directory"
self._package.source_url = self._path.as_posix()
super(DirectoryDependency, self).__init__(
self._package.name,
self._package.version,
category=category,
optional=optional,
allows_prereleases=True,
)
def _get_info_from_sdist(
self, url): # type: (str) -> Dict[str, Union[str, List, None]]
self._log(
"Downloading sdist: {}".format(
urlparse.urlparse(url).path.rsplit("/")[-1]),
level="debug",
)
info = {"summary": "", "requires_python": None, "requires_dist": None}
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
try:
meta = pkginfo.SDist(str(filepath))
if meta.summary:
info["summary"] = meta.summary
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
return info
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = filepath.suffix
gz = None
if suffix == ".zip":
tar = zipfile.ZipFile(str(filepath))
else:
if suffix == ".bz2":
gz = BZ2File(str(filepath))
suffixes = filepath.suffixes
if len(suffixes) > 1 and suffixes[-2] == ".tar":
suffix = ".tar.bz2"
else:
gz = GzipFile(str(filepath))
suffix = ".tar.gz"
tar = tarfile.TarFile(str(filepath), fileobj=gz)
try:
tar.extractall(os.path.join(temp_dir, "unpacked"))
finally:
if gz:
gz.close()
tar.close()
unpacked = Path(temp_dir) / "unpacked"
sdist_dir = unpacked / Path(filename).name.rstrip(suffix)
# Checking for .egg-info at root
eggs = list(sdist_dir.glob("*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Searching for .egg-info in sub directories
eggs = list(sdist_dir.glob("**/*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Still nothing, try reading (without executing it)
# the setup.py file.
try:
setup_info = self._inspect_sdist_with_setup(sdist_dir)
for key, value in info.items():
if value:
continue
info[key] = setup_info[key]
return info
except Exception as e:
self._log(
"An error occurred when reading setup.py or setup.cfg: {}".
format(str(e)),
"warning",
)
return info
def search_for_directory(
self, dependency): # type: (DirectoryDependency) -> List[Package]
if dependency.supports_poetry():
from poetry.poetry import Poetry
poetry = Poetry.create(dependency.full_path)
pkg = poetry.package
package = Package(pkg.name, pkg.version)
for dep in pkg.requires:
if not dep.is_optional():
package.requires.append(dep)
for extra, deps in pkg.extras.items():
if extra not in package.extras:
package.extras[extra] = []
for dep in deps:
package.extras[extra].append(dep)
package.python_versions = pkg.python_versions
else:
# Execute egg_info
current_dir = os.getcwd()
os.chdir(str(dependency.full_path))
try:
cwd = dependency.full_path
venv = Env.get(NullIO(), cwd=cwd)
venv.run("python", "setup.py", "egg_info")
except EnvCommandError:
result = SetupReader.read_from_directory(dependency.full_path)
if not result["name"]:
# The name could not be determined
# We use the dependency name
result["name"] = dependency.name
if not result["version"]:
# The version could not be determined
# so we raise an error since it is mandatory
raise RuntimeError(
"Unable to retrieve the package version for {}".format(
dependency.path))
package_name = result["name"]
package_version = result["version"]
python_requires = result["python_requires"]
if python_requires is None:
python_requires = "*"
package_summary = ""
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
reqs = parse_requires(requires)
else:
os.chdir(current_dir)
# Sometimes pathlib will fail on recursive
# symbolic links, so we need to workaround it
# and use the glob module instead.
# Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
if PY35:
egg_info = next(
Path(p) for p in glob.glob(
os.path.join(str(dependency.full_path), "**",
"*.egg-info"),
recursive=True,
))
else:
egg_info = next(dependency.full_path.glob("**/*.egg-info"))
meta = pkginfo.UnpackedSDist(str(egg_info))
package_name = meta.name
package_version = meta.version
package_summary = meta.summary
python_requires = meta.requires_python
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
reqs = parse_requires(f.read())
finally:
os.chdir(current_dir)
package = Package(package_name, package_version)
if dependency.name != package.name:
# For now, the dependency's name must match the actual package's name
raise RuntimeError(
"The dependency name for {} does not match the actual package's name: {}"
.format(dependency.name, package.name))
package.description = package_summary
for req in reqs:
dep = dependency_from_pep_508(req)
if dep.in_extras:
for extra in dep.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dep)
if not dep.is_optional():
package.requires.append(dep)
if python_requires:
package.python_versions = python_requires
package.source_type = "directory"
package.source_url = dependency.path.as_posix()
for extra in dependency.extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
return [package]
def __init__(
self,
path, # type: Path
category="main", # type: str
optional=False, # type: bool
base=None, # type: Path
develop=True, # type: bool
):
from . import dependency_from_pep_508
from .package import Package
self._path = path
self._base = base
self._full_path = path
self._develop = develop
self._supports_poetry = False
if self._base and not self._path.is_absolute():
self._full_path = self._base / self._path
if not self._full_path.exists():
raise ValueError("Directory {} does not exist".format(self._path))
if self._full_path.is_file():
raise ValueError("{} is a file, expected a directory".format(
self._path))
# Checking content to dertermine actions
setup = self._full_path / "setup.py"
pyproject = TomlFile(self._full_path / "pyproject.toml")
if pyproject.exists():
pyproject_content = pyproject.read()
self._supports_poetry = ("tool" in pyproject_content
and "poetry" in pyproject_content["tool"])
if not setup.exists() and not self._supports_poetry:
raise ValueError(
"Directory {} does not seem to be a Python package".format(
self._full_path))
if self._supports_poetry:
from poetry.poetry import Poetry
poetry = Poetry.create(self._full_path)
package = poetry.package
self._package = Package(package.pretty_name, package.version)
self._package.requires += package.requires
self._package.dev_requires += package.dev_requires
self._package.extras = package.extras
self._package.python_versions = package.python_versions
else:
# Execute egg_info
current_dir = os.getcwd()
os.chdir(str(self._full_path))
try:
cwd = base
venv = Env.get(NullIO(), cwd=cwd)
venv.run("python", "setup.py", "egg_info")
except EnvCommandError:
result = SetupReader.read_from_directory(self._full_path)
if not result["name"]:
# The name could not be determined
# so we raise an error since it is mandatory
raise RuntimeError(
"Unable to retrieve the package name for {}".format(
path))
if not result["version"]:
# The version could not be determined
# so we raise an error since it is mandatory
raise RuntimeError(
"Unable to retrieve the package version for {}".format(
path))
package_name = result["name"]
package_version = result["version"]
python_requires = result["python_requires"]
if python_requires is None:
python_requires = "*"
package_summary = ""
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
reqs = parse_requires(requires)
else:
os.chdir(current_dir)
# Sometimes pathlib will fail on recursive
# symbolic links, so we need to workaround it
# and use the glob module instead.
# Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
if PY35:
egg_info = next(
Path(p) for p in glob.glob(
os.path.join(str(self._full_path), "**",
"*.egg-info"),
recursive=True,
))
else:
egg_info = next(self._full_path.glob("**/*.egg-info"))
meta = pkginfo.UnpackedSDist(str(egg_info))
package_name = meta.name
package_version = meta.version
package_summary = meta.summary
python_requires = meta.requires_python
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open() as f:
reqs = parse_requires(f.read())
finally:
os.chdir(current_dir)
package = Package(package_name, package_version)
package.description = package_summary
for req in reqs:
package.requires.append(dependency_from_pep_508(req))
if python_requires:
package.python_versions = python_requires
self._package = package
self._package.source_type = "directory"
self._package.source_url = self._path.as_posix()
super(DirectoryDependency, self).__init__(
self._package.name,
self._package.version,
category=category,
optional=optional,
allows_prereleases=True,
)
def _get_requires_dist_from_sdist(self, url
): # type: (str) -> Union[list, None]
filename = os.path.basename(urlparse.urlparse(url).path)
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
try:
meta = pkginfo.SDist(str(filepath))
if meta.requires_dist:
return meta.requires_dist
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = filepath.suffix
gz = None
if suffix == '.zip':
tar = zipfile.ZipFile(str(filepath))
else:
if suffix == '.bz2':
gz = BZ2File(str(filepath))
else:
gz = GzipFile(str(filepath))
tar = tarfile.TarFile(str(filepath), fileobj=gz)
try:
tar.extractall(os.path.join(temp_dir, 'unpacked'))
finally:
if gz:
gz.close()
tar.close()
unpacked = Path(temp_dir) / 'unpacked'
sdist_dir = unpacked / Path(filename).name.rstrip('.tar.gz')
# Checking for .egg-info
eggs = list(sdist_dir.glob('*.egg-info'))
if eggs:
egg_info = eggs[0]
requires = egg_info / 'requires.txt'
if requires.exists():
with requires.open() as f:
return parse_requires(f.read())
return
# Still nothing, assume no dependencies
# We could probably get them by executing
# python setup.py egg-info but I don't feel
# confortable executing a file just for the sake
# of getting dependencies.
return
def get_package_from_directory(
cls,
directory,
name=None): # type: (Path, Optional[str]) -> Package
supports_poetry = False
pyproject = directory.joinpath("pyproject.toml")
if pyproject.exists():
pyproject = TomlFile(pyproject)
pyproject_content = pyproject.read()
supports_poetry = ("tool" in pyproject_content
and "poetry" in pyproject_content["tool"])
if supports_poetry:
poetry = Factory().create_poetry(directory)
pkg = poetry.package
package = Package(pkg.name, pkg.version)
for dep in pkg.requires:
if not dep.is_optional():
package.requires.append(dep)
for extra, deps in pkg.extras.items():
if extra not in package.extras:
package.extras[extra] = []
for dep in deps:
package.extras[extra].append(dep)
package.python_versions = pkg.python_versions
else:
# Execute egg_info
current_dir = os.getcwd()
os.chdir(str(directory))
try:
cwd = directory
venv = EnvManager().get(cwd)
venv.run("python", "setup.py", "egg_info")
except EnvCommandError:
result = SetupReader.read_from_directory(directory)
if not result["name"]:
# The name could not be determined
# We use the dependency name
result["name"] = name
if not result["version"]:
# The version could not be determined
# so we raise an error since it is mandatory
raise RuntimeError(
"Unable to retrieve the package version for {}".format(
directory))
package_name = result["name"]
package_version = result["version"]
python_requires = result["python_requires"]
if python_requires is None:
python_requires = "*"
package_summary = ""
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
reqs = parse_requires(requires)
else:
os.chdir(current_dir)
# Sometimes pathlib will fail on recursive
# symbolic links, so we need to workaround it
# and use the glob module instead.
# Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
if PY35:
egg_info = next(
Path(p) for p in glob.glob(
os.path.join(str(directory), "**", "*.egg-info"),
recursive=True,
))
else:
egg_info = next(directory.glob("**/*.egg-info"))
meta = pkginfo.UnpackedSDist(str(egg_info))
package_name = meta.name
package_version = meta.version
package_summary = meta.summary
python_requires = meta.requires_python
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
reqs = parse_requires(f.read())
finally:
os.chdir(current_dir)
package = Package(package_name, package_version)
package.description = package_summary
for req in reqs:
dep = dependency_from_pep_508(req)
if dep.in_extras:
for extra in dep.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dep)
if not dep.is_optional():
package.requires.append(dep)
if python_requires:
package.python_versions = python_requires
if name and name != package.name:
# For now, the dependency's name must match the actual package's name
raise RuntimeError(
"The dependency name for {} does not match the actual package's name: {}"
.format(name, package.name))
package.source_type = "directory"
package.source_url = directory.as_posix()
return package
def __init__(
self,
path, # type: Path
category='main', # type: str
optional=False, # type: bool
base=None, # type: Path
develop=False # type: bool
):
from . import dependency_from_pep_508
self._path = path
self._base = base
self._full_path = path
self._develop = develop
if self._base and not self._path.is_absolute():
self._full_path = self._base / self._path
if not self._full_path.exists():
raise ValueError('Directory {} does not exist'.format(self._path))
if self._full_path.is_file():
raise ValueError('{} is a file, expected a directory'.format(
self._path))
# Checking content to dertermine actions
setup = self._full_path / 'setup.py'
pyproject = TomlFile(self._full_path / 'pyproject.toml')
has_poetry = False
if pyproject.exists():
pyproject_content = pyproject.read(True)
has_poetry = ('tool' in pyproject_content
and 'poetry' in pyproject_content['tool'])
if not setup.exists() and not has_poetry:
raise ValueError(
'Directory {} does not seem to be a Python package'.format(
self._full_path))
if has_poetry:
from poetry.masonry.builders import SdistBuilder
from poetry.poetry import Poetry
poetry = Poetry.create(self._full_path)
builder = SdistBuilder(poetry, NullVenv(), NullIO())
with setup.open('w') as f:
f.write(decode(builder.build_setup()))
self._package = poetry.package
else:
from poetry.packages import Package
# Execute egg_info
current_dir = os.getcwd()
os.chdir(str(self._full_path))
try:
cwd = base
venv = Venv.create(NullIO(), cwd=cwd)
venv.run('python', 'setup.py', 'egg_info')
finally:
os.chdir(current_dir)
egg_info = list(self._full_path.glob('*.egg-info'))[0]
meta = pkginfo.UnpackedSDist(str(egg_info))
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / 'requires.txt'
if requires.exists():
with requires.open() as f:
reqs = parse_requires(f.read())
package = Package(meta.name, meta.version)
package.description = meta.summary
for req in reqs:
package.requires.append(dependency_from_pep_508(req))
if meta.requires_python:
package.python_versions = meta.requires_python
if meta.platforms:
platforms = [
p for p in meta.platforms if p.lower() != 'unknown'
]
if platforms:
package.platform = ' || '.join(platforms)
self._package = package
self._package.source_type = 'directory'
self._package.source_reference = str(self._path)
super(DirectoryDependency, self).__init__(self._package.name,
self._package.version,
category=category,
optional=optional,
allows_prereleases=True)
