def _inspect_sdist_with_setup(self, sdist_dir): info = {"requires_python": None, "requires_dist": None} setup = sdist_dir / "setup.py" if not setup.exists(): return info venv = Venv.create(NullIO()) current_dir = os.getcwd() os.chdir(sdist_dir.as_posix()) try: venv.run("python", "setup.py", "egg_info") egg_info = list(sdist_dir.glob("**/*.egg-info"))[0] meta = pkginfo.UnpackedSDist(str(egg_info)) if meta.requires_python: info["requires_python"] = meta.requires_python if meta.requires_dist: info["requires_dist"] = list(meta.requires_dist) else: requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: info["requires_dist"] = parse_requires(f.read()) except Exception: pass os.chdir(current_dir) return info
def test_parse_requires(): requires = """\ jsonschema>=2.6.0.0,<3.0.0.0 lockfile>=0.12.0.0,<0.13.0.0 pip-tools>=1.11.0.0,<2.0.0.0 pkginfo>=1.4.0.0,<2.0.0.0 pyrsistent>=0.14.2.0,<0.15.0.0 toml>=0.9.0.0,<0.10.0.0 cleo>=0.6.0.0,<0.7.0.0 cachy>=0.1.1.0,<0.2.0.0 cachecontrol>=0.12.4.0,<0.13.0.0 requests>=2.18.0.0,<3.0.0.0 msgpack-python>=0.5.0.0,<0.6.0.0 pyparsing>=2.2.0.0,<3.0.0.0 requests-toolbelt>=0.8.0.0,<0.9.0.0 [:(python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")] typing>=3.6.0.0,<4.0.0.0 [:python_version >= "2.7.0.0" and python_version < "2.8.0.0"] virtualenv>=15.2.0.0,<16.0.0.0 pathlib2>=2.3.0.0,<3.0.0.0 [:python_version >= "3.4.0.0" and python_version < "3.6.0.0"] zipfile36>=0.1.0.0,<0.2.0.0 [dev] isort@ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort """ result = parse_requires(requires) expected = [ "jsonschema>=2.6.0.0,<3.0.0.0", "lockfile>=0.12.0.0,<0.13.0.0", "pip-tools>=1.11.0.0,<2.0.0.0", "pkginfo>=1.4.0.0,<2.0.0.0", "pyrsistent>=0.14.2.0,<0.15.0.0", "toml>=0.9.0.0,<0.10.0.0", "cleo>=0.6.0.0,<0.7.0.0", "cachy>=0.1.1.0,<0.2.0.0", "cachecontrol>=0.12.4.0,<0.13.0.0", "requests>=2.18.0.0,<3.0.0.0", "msgpack-python>=0.5.0.0,<0.6.0.0", "pyparsing>=2.2.0.0,<3.0.0.0", "requests-toolbelt>=0.8.0.0,<0.9.0.0", 'typing>=3.6.0.0,<4.0.0.0 ; (python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")', 'virtualenv>=15.2.0.0,<16.0.0.0 ; python_version >= "2.7.0.0" and python_version < "2.8.0.0"', 'pathlib2>=2.3.0.0,<3.0.0.0 ; python_version >= "2.7.0.0" and python_version < "2.8.0.0"', 'zipfile36>=0.1.0.0,<0.2.0.0 ; python_version >= "3.4.0.0" and python_version < "3.6.0.0"', 'isort@ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort ; extra == "dev"', ] assert result == expected
def test_parse_requires(): requires = """\ jsonschema>=2.6.0.0,<3.0.0.0 lockfile>=0.12.0.0,<0.13.0.0 pip-tools>=1.11.0.0,<2.0.0.0 pkginfo>=1.4.0.0,<2.0.0.0 pyrsistent>=0.14.2.0,<0.15.0.0 toml>=0.9.0.0,<0.10.0.0 cleo>=0.6.0.0,<0.7.0.0 cachy>=0.1.1.0,<0.2.0.0 cachecontrol>=0.12.4.0,<0.13.0.0 requests>=2.18.0.0,<3.0.0.0 msgpack-python>=0.5.0.0,<0.6.0.0 pyparsing>=2.2.0.0,<3.0.0.0 requests-toolbelt>=0.8.0.0,<0.9.0.0 [:(python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")] typing>=3.6.0.0,<4.0.0.0 [:python_version >= "2.7.0.0" and python_version < "2.8.0.0"] virtualenv>=15.2.0.0,<16.0.0.0 pathlib2>=2.3.0.0,<3.0.0.0 [:python_version >= "3.4.0.0" and python_version < "3.6.0.0"] zipfile36>=0.1.0.0,<0.2.0.0 """ result = parse_requires(requires) expected = [ "jsonschema>=2.6.0.0,<3.0.0.0", "lockfile>=0.12.0.0,<0.13.0.0", "pip-tools>=1.11.0.0,<2.0.0.0", "pkginfo>=1.4.0.0,<2.0.0.0", "pyrsistent>=0.14.2.0,<0.15.0.0", "toml>=0.9.0.0,<0.10.0.0", "cleo>=0.6.0.0,<0.7.0.0", "cachy>=0.1.1.0,<0.2.0.0", "cachecontrol>=0.12.4.0,<0.13.0.0", "requests>=2.18.0.0,<3.0.0.0", "msgpack-python>=0.5.0.0,<0.6.0.0", "pyparsing>=2.2.0.0,<3.0.0.0", "requests-toolbelt>=0.8.0.0,<0.9.0.0", 'typing>=3.6.0.0,<4.0.0.0; (python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")', 'virtualenv>=15.2.0.0,<16.0.0.0; python_version >= "2.7.0.0" and python_version < "2.8.0.0"', 'pathlib2>=2.3.0.0,<3.0.0.0; python_version >= "2.7.0.0" and python_version < "2.8.0.0"', 'zipfile36>=0.1.0.0,<0.2.0.0; python_version >= "3.4.0.0" and python_version < "3.6.0.0"', ] assert result == expected
def _inspect_sdist_with_setup(self, sdist_dir): info = {"requires_python": None, "requires_dist": None} result = SetupReader.read_from_directory(sdist_dir) requires = "" for dep in result["install_requires"]: requires += dep + "\n" if result["extras_require"]: requires += "\n" for extra_name, deps in result["extras_require"].items(): requires += "[{}]\n".format(extra_name) for dep in deps: requires += dep + "\n" requires += "\n" info["requires_dist"] = parse_requires(requires) info["requires_python"] = result["python_requires"] return info
def _inspect_sdist_with_setup(self, sdist_dir): info = { 'requires_python': None, 'requires_dist': None, } setup = sdist_dir / 'setup.py' if not setup.exists(): return info venv = Venv.create(NullIO()) current_dir = os.getcwd() os.chdir(sdist_dir.as_posix()) try: venv.run('python', 'setup.py', 'egg_info') egg_info = list(sdist_dir.glob('**/*.egg-info'))[0] meta = pkginfo.UnpackedSDist(str(egg_info)) if meta.requires_python: info['requires_python'] = meta.requires_python if meta.requires_dist: info['requires_dist'] = list(meta.requires_dist) else: requires = egg_info / 'requires.txt' if requires.exists(): with requires.open() as f: info['requires_dist'] = parse_requires(f.read()) except Exception: pass os.chdir(current_dir) return info
def search_for_vcs(self, dependency): # type: (VCSDependency) -> List[Package] """ Search for the specifications that match the given VCS dependency. Basically, we clone the repository in a temporary directory and get the information we need by checking out the specified reference. """ if dependency.vcs != "git": raise ValueError("Unsupported VCS dependency {}".format( dependency.vcs)) tmp_dir = Path( mkdtemp(prefix="pypoetry-git-{}".format(dependency.name))) try: git = Git() git.clone(dependency.source, tmp_dir) git.checkout(dependency.reference, tmp_dir) revision = git.rev_parse(dependency.reference, tmp_dir).strip() if dependency.tag or dependency.rev: revision = dependency.reference pyproject = TomlFile(tmp_dir / "pyproject.toml") pyproject_content = None has_poetry = False if pyproject.exists(): pyproject_content = pyproject.read() has_poetry = ("tool" in pyproject_content and "poetry" in pyproject_content["tool"]) if pyproject_content and has_poetry: # If a pyproject.toml file exists # We use it to get the information we need info = pyproject_content["tool"]["poetry"] name = info["name"] version = info["version"] package = Package(name, version, version) package.source_type = dependency.vcs package.source_url = dependency.source package.source_reference = dependency.reference for req_name, req_constraint in info["dependencies"].items(): if req_name == "python": package.python_versions = req_constraint continue package.add_dependency(req_name, req_constraint) else: # We need to use setup.py here # to figure the information we need # We need to place ourselves in the proper # folder for it to work venv = Venv.create(self._io) current_dir = os.getcwd() os.chdir(tmp_dir.as_posix()) try: venv.run("python", "setup.py", "egg_info") # Sometimes pathlib will fail on recursive # symbolic links, so we need to workaround it # and use the glob module instead. # Note that this does not happen with pathlib2 # so it's safe to use it for Python < 3.4. if PY35: egg_info = next( Path(p) for p in glob.glob( os.path.join(str(tmp_dir), "**", "*.egg-info"), recursive=True, )) else: egg_info = next(tmp_dir.glob("**/*.egg-info")) meta = pkginfo.UnpackedSDist(str(egg_info)) if meta.requires_dist: reqs = list(meta.requires_dist) else: reqs = [] requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: reqs = parse_requires(f.read()) package = Package(meta.name, meta.version) for req in reqs: dep = dependency_from_pep_508(req) if dep.in_extras: for extra in dep.in_extras: if extra not in package.extras: package.extras[extra] = [] package.extras[extra].append(dep) package.requires.append(dep) except Exception: raise finally: os.chdir(current_dir) package.source_type = "git" package.source_url = dependency.source package.source_reference = revision except Exception: raise finally: shutil.rmtree(tmp_dir.as_posix()) if dependency.name != package.name: # For now, the dependency's name must match the actual package's name raise RuntimeError( "The dependency name for {} does not match the actual package's name: {}" .format(dependency.name, package.name)) if dependency.extras: for extra in dependency.extras: if extra in package.extras: for dep in package.extras[extra]: dep.activate() return [package]
def search_for_vcs(self, dependency): # type: (VCSDependency) -> List[Package] """ Search for the specifications that match the given VCS dependency. Basically, we clone the repository in a temporary directory and get the information we need by checking out the specified reference. """ if dependency.vcs != 'git': raise ValueError('Unsupported VCS dependency {}'.format( dependency.vcs)) tmp_dir = Path( mkdtemp(prefix='pypoetry-git-{}'.format(dependency.name))) try: git = Git() git.clone(dependency.source, tmp_dir) git.checkout(dependency.reference, tmp_dir) revision = git.rev_parse(dependency.reference, tmp_dir).strip() if dependency.tag or dependency.rev: revision = dependency.reference pyproject = TomlFile(tmp_dir / 'pyproject.toml') pyproject_content = None has_poetry = False if pyproject.exists(): pyproject_content = pyproject.read(True) has_poetry = ('tool' in pyproject_content and 'poetry' in pyproject_content['tool']) if pyproject_content and has_poetry: # If a pyproject.toml file exists # We use it to get the information we need info = pyproject_content['tool']['poetry'] name = info['name'] version = info['version'] package = Package(name, version, version) package.source_type = dependency.vcs package.source_url = dependency.source package.source_reference = dependency.reference for req_name, req_constraint in info['dependencies'].items(): if req_name == 'python': package.python_versions = req_constraint continue package.add_dependency(req_name, req_constraint) else: # We need to use setup.py here # to figure the information we need # We need to place ourselves in the proper # folder for it to work venv = Venv.create(self._io) current_dir = os.getcwd() os.chdir(tmp_dir.as_posix()) try: venv.run('python', 'setup.py', 'egg_info') egg_info = list(tmp_dir.glob('*.egg-info'))[0] meta = pkginfo.UnpackedSDist(str(egg_info)) if meta.requires_dist: reqs = list(meta.requires_dist) else: reqs = [] requires = egg_info / 'requires.txt' if requires.exists(): with requires.open() as f: reqs = parse_requires(f.read()) package = Package(meta.name, meta.version) for req in reqs: package.requires.append(dependency_from_pep_508(req)) except Exception: raise finally: os.chdir(current_dir) package.source_type = 'git' package.source_url = dependency.source package.source_reference = revision except Exception: raise finally: shutil.rmtree(tmp_dir.as_posix()) return [package]
def _get_info_from_sdist( self, url ): # type: (str) -> Dict[str, Union[str, List, None]] info = {"summary": "", "requires_python": None, "requires_dist": None} filename = os.path.basename(urlparse.urlparse(url).path) with temporary_directory() as temp_dir: filepath = Path(temp_dir) / filename self._download(url, str(filepath)) try: meta = pkginfo.SDist(str(filepath)) if meta.summary: info["summary"] = meta.summary if meta.requires_python: info["requires_python"] = meta.requires_python if meta.requires_dist: info["requires_dist"] = list(meta.requires_dist) return info except ValueError: # Unable to determine dependencies # We pass and go deeper pass # Still not dependencies found # So, we unpack and introspect suffix = filepath.suffix gz = None if suffix == ".zip": tar = zipfile.ZipFile(str(filepath)) else: if suffix == ".bz2": gz = BZ2File(str(filepath)) else: gz = GzipFile(str(filepath)) tar = tarfile.TarFile(str(filepath), fileobj=gz) try: tar.extractall(os.path.join(temp_dir, "unpacked")) finally: if gz: gz.close() tar.close() unpacked = Path(temp_dir) / "unpacked" sdist_dir = unpacked / Path(filename).name.rstrip(".tar.gz") # Checking for .egg-info at root eggs = list(sdist_dir.glob("*.egg-info")) if eggs: egg_info = eggs[0] requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: info["requires_dist"] = parse_requires(f.read()) return info # Searching for .egg-info in sub directories eggs = list(sdist_dir.glob("**/*.egg-info")) if eggs: egg_info = eggs[0] requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: info["requires_dist"] = parse_requires(f.read()) return info # Still nothing, assume no dependencies # We could probably get them by executing # python setup.py egg-info but I don't feel # confortable executing a file just for the sake # of getting dependencies. return info
def _get_info_from_sdist( self, url): # type: (str) -> Dict[str, Union[str, List, None]] info = {"summary": "", "requires_python": None, "requires_dist": None} filename = os.path.basename(urlparse.urlparse(url).path) with temporary_directory() as temp_dir: filepath = Path(temp_dir) / filename self._download(url, str(filepath)) try: meta = pkginfo.SDist(str(filepath)) if meta.summary: info["summary"] = meta.summary if meta.requires_python: info["requires_python"] = meta.requires_python if meta.requires_dist: info["requires_dist"] = list(meta.requires_dist) return info except ValueError: # Unable to determine dependencies # We pass and go deeper pass # Still not dependencies found # So, we unpack and introspect suffix = filepath.suffix gz = None if suffix == ".zip": tar = zipfile.ZipFile(str(filepath)) else: if suffix == ".bz2": gz = BZ2File(str(filepath)) else: gz = GzipFile(str(filepath)) suffix = ".tar.gz" tar = tarfile.TarFile(str(filepath), fileobj=gz) try: tar.extractall(os.path.join(temp_dir, "unpacked")) finally: if gz: gz.close() tar.close() unpacked = Path(temp_dir) / "unpacked" sdist_dir = unpacked / Path(filename).name.rstrip(suffix) # Checking for .egg-info at root eggs = list(sdist_dir.glob("*.egg-info")) if eggs: egg_info = eggs[0] requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: info["requires_dist"] = parse_requires(f.read()) return info # Searching for .egg-info in sub directories eggs = list(sdist_dir.glob("**/*.egg-info")) if eggs: egg_info = eggs[0] requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: info["requires_dist"] = parse_requires(f.read()) return info # Still nothing, assume no dependencies # We could probably get them by executing # python setup.py egg-info but I don't feel # confortable executing a file just for the sake # of getting dependencies. return info
def search_for_vcs(self, dependency): # type: (VCSDependency) -> List[Package] """ Search for the specifications that match the given VCS dependency. Basically, we clone the repository in a temporary directory and get the information we need by checking out the specified reference. """ if dependency.vcs != "git": raise ValueError("Unsupported VCS dependency {}".format(dependency.vcs)) tmp_dir = Path(mkdtemp(prefix="pypoetry-git-{}".format(dependency.name))) try: git = Git() git.clone(dependency.source, tmp_dir) git.checkout(dependency.reference, tmp_dir) revision = git.rev_parse(dependency.reference, tmp_dir).strip() if dependency.tag or dependency.rev: revision = dependency.reference pyproject = TomlFile(tmp_dir / "pyproject.toml") pyproject_content = None has_poetry = False if pyproject.exists(): pyproject_content = pyproject.read() has_poetry = ( "tool" in pyproject_content and "poetry" in pyproject_content["tool"] ) if pyproject_content and has_poetry: # If a pyproject.toml file exists # We use it to get the information we need info = pyproject_content["tool"]["poetry"] name = info["name"] version = info["version"] package = Package(name, version, version) package.source_type = dependency.vcs package.source_url = dependency.source package.source_reference = dependency.reference for req_name, req_constraint in info["dependencies"].items(): if req_name == "python": package.python_versions = req_constraint continue package.add_dependency(req_name, req_constraint) else: # We need to use setup.py here # to figure the information we need # We need to place ourselves in the proper # folder for it to work venv = Venv.create(self._io) current_dir = os.getcwd() os.chdir(tmp_dir.as_posix()) try: venv.run("python", "setup.py", "egg_info") # Sometimes pathlib will fail on recursive # symbolic links, so we need to workaround it # and use the glob module instead. # Note that this does not happen with pathlib2 # so it's safe to use it for Python < 3.4. if PY35: egg_info = next( Path(p) for p in glob.glob( os.path.join(str(tmp_dir), "**", "*.egg-info"), recursive=True, ) ) else: egg_info = next(tmp_dir.glob("**/*.egg-info")) meta = pkginfo.UnpackedSDist(str(egg_info)) if meta.requires_dist: reqs = list(meta.requires_dist) else: reqs = [] requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: reqs = parse_requires(f.read()) package = Package(meta.name, meta.version) for req in reqs: dep = dependency_from_pep_508(req) if dep.in_extras: for extra in dep.in_extras: if extra not in package.extras: package.extras[extra] = [] package.extras[extra].append(dep) package.requires.append(dep) except Exception: raise finally: os.chdir(current_dir) package.source_type = "git" package.source_url = dependency.source package.source_reference = revision except Exception: raise finally: shutil.rmtree(tmp_dir.as_posix()) if dependency.name != package.name: # For now, the dependency's name must match the actual package's name raise RuntimeError( "The dependency name for {} does not match the actual package's name: {}".format( dependency.name, package.name ) ) if dependency.extras: for extra in dependency.extras: if extra in package.extras: for dep in package.extras[extra]: dep.activate() return [package]
def __init__( self, path, # type: Path category="main", # type: str optional=False, # type: bool base=None, # type: Path develop=False, # type: bool ): from . import dependency_from_pep_508 from .package import Package self._path = path self._base = base self._full_path = path self._develop = develop if self._base and not self._path.is_absolute(): self._full_path = self._base / self._path if not self._full_path.exists(): raise ValueError("Directory {} does not exist".format(self._path)) if self._full_path.is_file(): raise ValueError("{} is a file, expected a directory".format( self._path)) # Checking content to dertermine actions setup = self._full_path / "setup.py" pyproject = TomlFile(self._full_path / "pyproject.toml") has_poetry = False if pyproject.exists(): pyproject_content = pyproject.read() has_poetry = ("tool" in pyproject_content and "poetry" in pyproject_content["tool"]) if not setup.exists() and not has_poetry: raise ValueError( "Directory {} does not seem to be a Python package".format( self._full_path)) if has_poetry: from poetry.masonry.builders import SdistBuilder from poetry.poetry import Poetry poetry = Poetry.create(self._full_path) builder = SdistBuilder(poetry, NullVenv(), NullIO()) with setup.open("w") as f: f.write(decode(builder.build_setup())) package = poetry.package self._package = Package(package.pretty_name, package.version) self._package.requires += package.requires self._package.dev_requires += package.dev_requires self._package.extras = package.extras self._package.python_versions = package.python_versions self._package.platform = package.platform else: # Execute egg_info current_dir = os.getcwd() os.chdir(str(self._full_path)) try: cwd = base venv = Venv.create(NullIO(), cwd=cwd) venv.run("python", "setup.py", "egg_info") finally: os.chdir(current_dir) egg_info = list(self._full_path.glob("*.egg-info"))[0] meta = pkginfo.UnpackedSDist(str(egg_info)) if meta.requires_dist: reqs = list(meta.requires_dist) else: reqs = [] requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: reqs = parse_requires(f.read()) package = Package(meta.name, meta.version) package.description = meta.summary for req in reqs: package.requires.append(dependency_from_pep_508(req)) if meta.requires_python: package.python_versions = meta.requires_python if meta.platforms: platforms = [ p for p in meta.platforms if p.lower() != "unknown" ] if platforms: package.platform = " || ".join(platforms) self._package = package self._package.source_type = "directory" self._package.source_url = self._path.as_posix() super(DirectoryDependency, self).__init__( self._package.name, self._package.version, category=category, optional=optional, allows_prereleases=True, )
def __init__( self, path, # type: Path category="main", # type: str optional=False, # type: bool base=None, # type: Path develop=False, # type: bool ): from . import dependency_from_pep_508 from .package import Package self._path = path self._base = base self._full_path = path self._develop = develop if self._base and not self._path.is_absolute(): self._full_path = self._base / self._path if not self._full_path.exists(): raise ValueError("Directory {} does not exist".format(self._path)) if self._full_path.is_file(): raise ValueError("{} is a file, expected a directory".format(self._path)) # Checking content to dertermine actions setup = self._full_path / "setup.py" pyproject = TomlFile(self._full_path / "pyproject.toml") has_poetry = False if pyproject.exists(): pyproject_content = pyproject.read(True) has_poetry = ( "tool" in pyproject_content and "poetry" in pyproject_content["tool"] ) if not setup.exists() and not has_poetry: raise ValueError( "Directory {} does not seem to be a Python package".format( self._full_path ) ) if has_poetry: from poetry.masonry.builders import SdistBuilder from poetry.poetry import Poetry poetry = Poetry.create(self._full_path) builder = SdistBuilder(poetry, NullVenv(), NullIO()) with setup.open("w") as f: f.write(decode(builder.build_setup())) package = poetry.package self._package = Package(package.pretty_name, package.version) self._package.requires += package.requires self._package.dev_requires += package.dev_requires self._package.python_versions = package.python_versions self._package.platform = package.platform else: # Execute egg_info current_dir = os.getcwd() os.chdir(str(self._full_path)) try: cwd = base venv = Venv.create(NullIO(), cwd=cwd) venv.run("python", "setup.py", "egg_info") finally: os.chdir(current_dir) egg_info = list(self._full_path.glob("*.egg-info"))[0] meta = pkginfo.UnpackedSDist(str(egg_info)) if meta.requires_dist: reqs = list(meta.requires_dist) else: reqs = [] requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: reqs = parse_requires(f.read()) package = Package(meta.name, meta.version) package.description = meta.summary for req in reqs: package.requires.append(dependency_from_pep_508(req)) if meta.requires_python: package.python_versions = meta.requires_python if meta.platforms: platforms = [p for p in meta.platforms if p.lower() != "unknown"] if platforms: package.platform = " || ".join(platforms) self._package = package self._package.source_type = "directory" self._package.source_url = self._path.as_posix() super(DirectoryDependency, self).__init__( self._package.name, self._package.version, category=category, optional=optional, allows_prereleases=True, )
def _get_info_from_sdist( self, url): # type: (str) -> Dict[str, Union[str, List, None]] self._log( "Downloading sdist: {}".format( urlparse.urlparse(url).path.rsplit("/")[-1]), level="debug", ) info = {"summary": "", "requires_python": None, "requires_dist": None} filename = os.path.basename(urlparse.urlparse(url).path) with temporary_directory() as temp_dir: filepath = Path(temp_dir) / filename self._download(url, str(filepath)) try: meta = pkginfo.SDist(str(filepath)) if meta.summary: info["summary"] = meta.summary if meta.requires_python: info["requires_python"] = meta.requires_python if meta.requires_dist: info["requires_dist"] = list(meta.requires_dist) return info except ValueError: # Unable to determine dependencies # We pass and go deeper pass # Still not dependencies found # So, we unpack and introspect suffix = filepath.suffix gz = None if suffix == ".zip": tar = zipfile.ZipFile(str(filepath)) else: if suffix == ".bz2": gz = BZ2File(str(filepath)) suffixes = filepath.suffixes if len(suffixes) > 1 and suffixes[-2] == ".tar": suffix = ".tar.bz2" else: gz = GzipFile(str(filepath)) suffix = ".tar.gz" tar = tarfile.TarFile(str(filepath), fileobj=gz) try: tar.extractall(os.path.join(temp_dir, "unpacked")) finally: if gz: gz.close() tar.close() unpacked = Path(temp_dir) / "unpacked" sdist_dir = unpacked / Path(filename).name.rstrip(suffix) # Checking for .egg-info at root eggs = list(sdist_dir.glob("*.egg-info")) if eggs: egg_info = eggs[0] requires = egg_info / "requires.txt" if requires.exists(): with requires.open(encoding="utf-8") as f: info["requires_dist"] = parse_requires(f.read()) return info # Searching for .egg-info in sub directories eggs = list(sdist_dir.glob("**/*.egg-info")) if eggs: egg_info = eggs[0] requires = egg_info / "requires.txt" if requires.exists(): with requires.open(encoding="utf-8") as f: info["requires_dist"] = parse_requires(f.read()) return info # Still nothing, try reading (without executing it) # the setup.py file. try: setup_info = self._inspect_sdist_with_setup(sdist_dir) for key, value in info.items(): if value: continue info[key] = setup_info[key] return info except Exception as e: self._log( "An error occurred when reading setup.py or setup.cfg: {}". format(str(e)), "warning", ) return info
def search_for_directory( self, dependency): # type: (DirectoryDependency) -> List[Package] if dependency.supports_poetry(): from poetry.poetry import Poetry poetry = Poetry.create(dependency.full_path) pkg = poetry.package package = Package(pkg.name, pkg.version) for dep in pkg.requires: if not dep.is_optional(): package.requires.append(dep) for extra, deps in pkg.extras.items(): if extra not in package.extras: package.extras[extra] = [] for dep in deps: package.extras[extra].append(dep) package.python_versions = pkg.python_versions else: # Execute egg_info current_dir = os.getcwd() os.chdir(str(dependency.full_path)) try: cwd = dependency.full_path venv = Env.get(NullIO(), cwd=cwd) venv.run("python", "setup.py", "egg_info") except EnvCommandError: result = SetupReader.read_from_directory(dependency.full_path) if not result["name"]: # The name could not be determined # We use the dependency name result["name"] = dependency.name if not result["version"]: # The version could not be determined # so we raise an error since it is mandatory raise RuntimeError( "Unable to retrieve the package version for {}".format( dependency.path)) package_name = result["name"] package_version = result["version"] python_requires = result["python_requires"] if python_requires is None: python_requires = "*" package_summary = "" requires = "" for dep in result["install_requires"]: requires += dep + "\n" if result["extras_require"]: requires += "\n" for extra_name, deps in result["extras_require"].items(): requires += "[{}]\n".format(extra_name) for dep in deps: requires += dep + "\n" requires += "\n" reqs = parse_requires(requires) else: os.chdir(current_dir) # Sometimes pathlib will fail on recursive # symbolic links, so we need to workaround it # and use the glob module instead. # Note that this does not happen with pathlib2 # so it's safe to use it for Python < 3.4. if PY35: egg_info = next( Path(p) for p in glob.glob( os.path.join(str(dependency.full_path), "**", "*.egg-info"), recursive=True, )) else: egg_info = next(dependency.full_path.glob("**/*.egg-info")) meta = pkginfo.UnpackedSDist(str(egg_info)) package_name = meta.name package_version = meta.version package_summary = meta.summary python_requires = meta.requires_python if meta.requires_dist: reqs = list(meta.requires_dist) else: reqs = [] requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: reqs = parse_requires(f.read()) finally: os.chdir(current_dir) package = Package(package_name, package_version) if dependency.name != package.name: # For now, the dependency's name must match the actual package's name raise RuntimeError( "The dependency name for {} does not match the actual package's name: {}" .format(dependency.name, package.name)) package.description = package_summary for req in reqs: dep = dependency_from_pep_508(req) if dep.in_extras: for extra in dep.in_extras: if extra not in package.extras: package.extras[extra] = [] package.extras[extra].append(dep) if not dep.is_optional(): package.requires.append(dep) if python_requires: package.python_versions = python_requires package.source_type = "directory" package.source_url = dependency.path.as_posix() for extra in dependency.extras: if extra in package.extras: for dep in package.extras[extra]: dep.activate() package.requires += package.extras[extra] return [package]
def __init__( self, path, # type: Path category="main", # type: str optional=False, # type: bool base=None, # type: Path develop=True, # type: bool ): from . import dependency_from_pep_508 from .package import Package self._path = path self._base = base self._full_path = path self._develop = develop self._supports_poetry = False if self._base and not self._path.is_absolute(): self._full_path = self._base / self._path if not self._full_path.exists(): raise ValueError("Directory {} does not exist".format(self._path)) if self._full_path.is_file(): raise ValueError("{} is a file, expected a directory".format( self._path)) # Checking content to dertermine actions setup = self._full_path / "setup.py" pyproject = TomlFile(self._full_path / "pyproject.toml") if pyproject.exists(): pyproject_content = pyproject.read() self._supports_poetry = ("tool" in pyproject_content and "poetry" in pyproject_content["tool"]) if not setup.exists() and not self._supports_poetry: raise ValueError( "Directory {} does not seem to be a Python package".format( self._full_path)) if self._supports_poetry: from poetry.poetry import Poetry poetry = Poetry.create(self._full_path) package = poetry.package self._package = Package(package.pretty_name, package.version) self._package.requires += package.requires self._package.dev_requires += package.dev_requires self._package.extras = package.extras self._package.python_versions = package.python_versions else: # Execute egg_info current_dir = os.getcwd() os.chdir(str(self._full_path)) try: cwd = base venv = Env.get(NullIO(), cwd=cwd) venv.run("python", "setup.py", "egg_info") except EnvCommandError: result = SetupReader.read_from_directory(self._full_path) if not result["name"]: # The name could not be determined # so we raise an error since it is mandatory raise RuntimeError( "Unable to retrieve the package name for {}".format( path)) if not result["version"]: # The version could not be determined # so we raise an error since it is mandatory raise RuntimeError( "Unable to retrieve the package version for {}".format( path)) package_name = result["name"] package_version = result["version"] python_requires = result["python_requires"] if python_requires is None: python_requires = "*" package_summary = "" requires = "" for dep in result["install_requires"]: requires += dep + "\n" if result["extras_require"]: requires += "\n" for extra_name, deps in result["extras_require"].items(): requires += "[{}]\n".format(extra_name) for dep in deps: requires += dep + "\n" requires += "\n" reqs = parse_requires(requires) else: os.chdir(current_dir) # Sometimes pathlib will fail on recursive # symbolic links, so we need to workaround it # and use the glob module instead. # Note that this does not happen with pathlib2 # so it's safe to use it for Python < 3.4. if PY35: egg_info = next( Path(p) for p in glob.glob( os.path.join(str(self._full_path), "**", "*.egg-info"), recursive=True, )) else: egg_info = next(self._full_path.glob("**/*.egg-info")) meta = pkginfo.UnpackedSDist(str(egg_info)) package_name = meta.name package_version = meta.version package_summary = meta.summary python_requires = meta.requires_python if meta.requires_dist: reqs = list(meta.requires_dist) else: reqs = [] requires = egg_info / "requires.txt" if requires.exists(): with requires.open() as f: reqs = parse_requires(f.read()) finally: os.chdir(current_dir) package = Package(package_name, package_version) package.description = package_summary for req in reqs: package.requires.append(dependency_from_pep_508(req)) if python_requires: package.python_versions = python_requires self._package = package self._package.source_type = "directory" self._package.source_url = self._path.as_posix() super(DirectoryDependency, self).__init__( self._package.name, self._package.version, category=category, optional=optional, allows_prereleases=True, )
def _get_requires_dist_from_sdist(self, url ): # type: (str) -> Union[list, None] filename = os.path.basename(urlparse.urlparse(url).path) with temporary_directory() as temp_dir: filepath = Path(temp_dir) / filename self._download(url, str(filepath)) try: meta = pkginfo.SDist(str(filepath)) if meta.requires_dist: return meta.requires_dist except ValueError: # Unable to determine dependencies # We pass and go deeper pass # Still not dependencies found # So, we unpack and introspect suffix = filepath.suffix gz = None if suffix == '.zip': tar = zipfile.ZipFile(str(filepath)) else: if suffix == '.bz2': gz = BZ2File(str(filepath)) else: gz = GzipFile(str(filepath)) tar = tarfile.TarFile(str(filepath), fileobj=gz) try: tar.extractall(os.path.join(temp_dir, 'unpacked')) finally: if gz: gz.close() tar.close() unpacked = Path(temp_dir) / 'unpacked' sdist_dir = unpacked / Path(filename).name.rstrip('.tar.gz') # Checking for .egg-info eggs = list(sdist_dir.glob('*.egg-info')) if eggs: egg_info = eggs[0] requires = egg_info / 'requires.txt' if requires.exists(): with requires.open() as f: return parse_requires(f.read()) return # Still nothing, assume no dependencies # We could probably get them by executing # python setup.py egg-info but I don't feel # confortable executing a file just for the sake # of getting dependencies. return
def get_package_from_directory( cls, directory, name=None): # type: (Path, Optional[str]) -> Package supports_poetry = False pyproject = directory.joinpath("pyproject.toml") if pyproject.exists(): pyproject = TomlFile(pyproject) pyproject_content = pyproject.read() supports_poetry = ("tool" in pyproject_content and "poetry" in pyproject_content["tool"]) if supports_poetry: poetry = Factory().create_poetry(directory) pkg = poetry.package package = Package(pkg.name, pkg.version) for dep in pkg.requires: if not dep.is_optional(): package.requires.append(dep) for extra, deps in pkg.extras.items(): if extra not in package.extras: package.extras[extra] = [] for dep in deps: package.extras[extra].append(dep) package.python_versions = pkg.python_versions else: # Execute egg_info current_dir = os.getcwd() os.chdir(str(directory)) try: cwd = directory venv = EnvManager().get(cwd) venv.run("python", "setup.py", "egg_info") except EnvCommandError: result = SetupReader.read_from_directory(directory) if not result["name"]: # The name could not be determined # We use the dependency name result["name"] = name if not result["version"]: # The version could not be determined # so we raise an error since it is mandatory raise RuntimeError( "Unable to retrieve the package version for {}".format( directory)) package_name = result["name"] package_version = result["version"] python_requires = result["python_requires"] if python_requires is None: python_requires = "*" package_summary = "" requires = "" for dep in result["install_requires"]: requires += dep + "\n" if result["extras_require"]: requires += "\n" for extra_name, deps in result["extras_require"].items(): requires += "[{}]\n".format(extra_name) for dep in deps: requires += dep + "\n" requires += "\n" reqs = parse_requires(requires) else: os.chdir(current_dir) # Sometimes pathlib will fail on recursive # symbolic links, so we need to workaround it # and use the glob module instead. # Note that this does not happen with pathlib2 # so it's safe to use it for Python < 3.4. if PY35: egg_info = next( Path(p) for p in glob.glob( os.path.join(str(directory), "**", "*.egg-info"), recursive=True, )) else: egg_info = next(directory.glob("**/*.egg-info")) meta = pkginfo.UnpackedSDist(str(egg_info)) package_name = meta.name package_version = meta.version package_summary = meta.summary python_requires = meta.requires_python if meta.requires_dist: reqs = list(meta.requires_dist) else: reqs = [] requires = egg_info / "requires.txt" if requires.exists(): with requires.open(encoding="utf-8") as f: reqs = parse_requires(f.read()) finally: os.chdir(current_dir) package = Package(package_name, package_version) package.description = package_summary for req in reqs: dep = dependency_from_pep_508(req) if dep.in_extras: for extra in dep.in_extras: if extra not in package.extras: package.extras[extra] = [] package.extras[extra].append(dep) if not dep.is_optional(): package.requires.append(dep) if python_requires: package.python_versions = python_requires if name and name != package.name: # For now, the dependency's name must match the actual package's name raise RuntimeError( "The dependency name for {} does not match the actual package's name: {}" .format(name, package.name)) package.source_type = "directory" package.source_url = directory.as_posix() return package
def __init__( self, path, # type: Path category='main', # type: str optional=False, # type: bool base=None, # type: Path develop=False # type: bool ): from . import dependency_from_pep_508 self._path = path self._base = base self._full_path = path self._develop = develop if self._base and not self._path.is_absolute(): self._full_path = self._base / self._path if not self._full_path.exists(): raise ValueError('Directory {} does not exist'.format(self._path)) if self._full_path.is_file(): raise ValueError('{} is a file, expected a directory'.format( self._path)) # Checking content to dertermine actions setup = self._full_path / 'setup.py' pyproject = TomlFile(self._full_path / 'pyproject.toml') has_poetry = False if pyproject.exists(): pyproject_content = pyproject.read(True) has_poetry = ('tool' in pyproject_content and 'poetry' in pyproject_content['tool']) if not setup.exists() and not has_poetry: raise ValueError( 'Directory {} does not seem to be a Python package'.format( self._full_path)) if has_poetry: from poetry.masonry.builders import SdistBuilder from poetry.poetry import Poetry poetry = Poetry.create(self._full_path) builder = SdistBuilder(poetry, NullVenv(), NullIO()) with setup.open('w') as f: f.write(decode(builder.build_setup())) self._package = poetry.package else: from poetry.packages import Package # Execute egg_info current_dir = os.getcwd() os.chdir(str(self._full_path)) try: cwd = base venv = Venv.create(NullIO(), cwd=cwd) venv.run('python', 'setup.py', 'egg_info') finally: os.chdir(current_dir) egg_info = list(self._full_path.glob('*.egg-info'))[0] meta = pkginfo.UnpackedSDist(str(egg_info)) if meta.requires_dist: reqs = list(meta.requires_dist) else: reqs = [] requires = egg_info / 'requires.txt' if requires.exists(): with requires.open() as f: reqs = parse_requires(f.read()) package = Package(meta.name, meta.version) package.description = meta.summary for req in reqs: package.requires.append(dependency_from_pep_508(req)) if meta.requires_python: package.python_versions = meta.requires_python if meta.platforms: platforms = [ p for p in meta.platforms if p.lower() != 'unknown' ] if platforms: package.platform = ' || '.join(platforms) self._package = package self._package.source_type = 'directory' self._package.source_reference = str(self._path) super(DirectoryDependency, self).__init__(self._package.name, self._package.version, category=category, optional=optional, allows_prereleases=True)