def test_parse_requirements_vcs(monkeypatch): requirement_text = "git+https://github.com/bar/foo" files = {"a.txt": [requirement_text + "\n"]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) with pytest.raises(PipError): pip_api.parse_requirements("a.txt")
def test_parse_requirements_missing_hashes_late(monkeypatch, strict_hashes): files = { "a.txt": [ "foo==1.2.3\n", "bar==1.2.3\n", "baz==1.2.3 --hash=sha256:862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e\n", ] } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) if strict_hashes: with pytest.raises( PipError, match=r"Missing hashes for requirement in a\.txt, line 1"): pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) else: result = pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes) assert result["foo"].hashes == {} assert result["bar"].hashes == {} assert result["baz"].hashes == { "sha256": [ "862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e" ], }
def test_parse_requirements_invalid_hash_kind(monkeypatch): files = { "a.txt": ["foo==1.2.3 --hash=md5:0d5a28f01dccb5a549c31016883f59c2"] } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) with pytest.raises(PipError, match=r"Invalid --hash kind"): pip_api.parse_requirements("a.txt")
def test_parse_requirements_double_raises(monkeypatch): files = {"a.txt": ["foo==1.2.3\n", "foo==3.2.1\n"]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) with pytest.raises(pip_api.exceptions.PipError) as e: pip_api.parse_requirements("a.txt") assert e.value.args == ( "Double requirement given: foo==3.2.1 (already in foo==1.2.3, name='foo')", )
def test_parse_requirements_with_invalid_wheel_filename(monkeypatch): INVALID_WHEEL_NAME = "pip-1.3.1-invalid-format.whl" files = { "a.txt": ["https://github.com/pypa/pip/archive/" + INVALID_WHEEL_NAME], } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) with pytest.raises(PipError, match=r"Invalid wheel name: " + INVALID_WHEEL_NAME): pip_api.parse_requirements("a.txt")
def test_parse_requirements_with_missing_egg_suffix(monkeypatch): # Without a package name, an `#egg=foo` suffix is required to know the package name files = { "a.txt": [PEP508_PIP_EXAMPLE_URL], } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) with pytest.raises(PipError, match=r"Missing egg fragment in URL: " + PEP508_PIP_EXAMPLE_URL): pip_api.parse_requirements("a.txt")
def test_parse_requirements_double_raises(monkeypatch): files = { 'a.txt': [ 'foo==1.2.3\n', 'foo==3.2.1\n', ], } monkeypatch.setattr(pip_api._parse_requirements, '_read_file', files.get) with pytest.raises(pip_api.exceptions.PipError) as e: pip_api.parse_requirements('a.txt') assert e.value.args == ( "Double requirement given: foo==3.2.1 (already in foo==1.2.3, name='foo')", # noqa )
def test_parse_requirements_missing_all_hashes_strict(monkeypatch): files = { "a.txt": [ "foo==1.2.3\n", "bar==1.2.3\n", "baz==1.2.3\n", ] } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) with pytest.raises( PipError, match=r"Missing hashes for requirement in a\.txt, line 1"): pip_api.parse_requirements("a.txt", strict_hashes=True)
def test_parse_requirements_hashes(monkeypatch): files = { "a.txt": [ "foo==1.2.3 " "--hash=sha256:862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e " "--hash=sha256:0cfea7e5a53d5a256b4e8609c8a1812ad9af5c611432ec9dccbb4d79dc6a336e " "--hash=sha384:673546e6c3236a36e5db5f1bc9d2cb5f3f974d3d4e9031f405b1dc7874575e2ad91436d02edf8237a889ab1cecb35d56 " "--hash=sha512:3b149832490a704091abed6a9bd40ef7f4176b279263d4cbbb440b067ced99cadc006c03bc47488755351022fb49f2f10edfec110f027039bda703d407135c47" ] } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt") assert set(result) == {"foo"} assert result["foo"].hashes == { "sha256": [ "862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e", "0cfea7e5a53d5a256b4e8609c8a1812ad9af5c611432ec9dccbb4d79dc6a336e", ], "sha384": [ "673546e6c3236a36e5db5f1bc9d2cb5f3f974d3d4e9031f405b1dc7874575e2ad91436d02edf8237a889ab1cecb35d56" ], "sha512": [ "3b149832490a704091abed6a9bd40ef7f4176b279263d4cbbb440b067ced99cadc006c03bc47488755351022fb49f2f10edfec110f027039bda703d407135c47" ], }
def test_parse_requirements_recursive(monkeypatch, flag): files = {"a.txt": ["{} b.txt\n".format(flag)], "b.txt": ["foo==1.2.3\n"]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt") assert set(result) == {"foo"} assert str(result["foo"]) == "foo==1.2.3"
def test_parse_requirements(monkeypatch): files = {"a.txt": ["foo==1.2.3\n"]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt") assert set(result) == {"foo"} assert str(result["foo"]) == "foo==1.2.3"
def test_parse_requirements_editable_file(monkeypatch): files = {"a.txt": ["Django==1.11\n" "-e .\n"]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt") assert set(result) == {"django", "pip-api"} assert str(result["django"]) == "Django==1.11" assert str(result["pip-api"]).startswith("pip-api@ file:///")
def test_parse_requirements_multiline(monkeypatch, lines): files = {"a.txt": lines, "b.txt": ["foo==1.2.3\n"]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.__getitem__) result = pip_api.parse_requirements("a.txt") assert set(result) == {"foo"} assert str(result["foo"]) == "foo==1.2.3"
def _get_names_cached(cls, path): results = [] with chdir(os.path.dirname(path)): requirements = parse_requirements(path) for req in requirements.values(): if req.name: results.append(req.name) return results
def get_requirements(self): try: return { packaging.utils.canonicalize_name(name): req for name, req in pip_api.parse_requirements( self.filename, options=PipOption(self.config)).items() } except pip_api.exceptions.PipError as e: raise ReqsError("%s (from -r %s)" % (e.args[0].split("\n")[0], self.filename))
def test_parse_requirements_with_relative_references(monkeypatch): files = { "reqs/base.txt": ["django==1.11\n"], "reqs/test.txt": ["-r base.txt\n"], "reqs/dev.txt": ["-r base.txt\n" "-r test.txt\n"], } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("reqs/dev.txt") assert set(result) == {"django"}
def test_parse_requirements_PEP508(monkeypatch, line, result_set, url, string, spec): files = {"a.txt": [line]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt") assert set(result) == result_set assert result["pip"].url == url assert str(result["pip"]) == string assert result["pip"].specifier == spec
def test_include_invalid_requirement(monkeypatch): requirement_text = "git+https://github.com/bar/foo" files = {"a.txt": [requirement_text + "\n"]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt", include_invalid=True) assert set(result) == {requirement_text} assert result[requirement_text].name == requirement_text assert (str(result[requirement_text]) == f"Missing egg fragment in URL: {requirement_text}")
def test_parse_requirements_with_index_url(monkeypatch, flag): files = { "a.txt": ["{} https://example.com/pypi/simple\n".format(flag), "foo==1.2.3\n"] } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt") assert set(result) == {"foo"} assert str(result["foo"]) == "foo==1.2.3"
def test_parse_requirements(monkeypatch): files = { 'a.txt': [ 'foo==1.2.3\n', ], } monkeypatch.setattr(pip_api._parse_requirements, '_read_file', files.get) result = pip_api.parse_requirements('a.txt') assert set(result) == {'foo'} assert str(result['foo']) == 'foo==1.2.3'
def get_requirements(self): try: return { packaging.utils.canonicalize_name(name): req for name, req in pip_api.parse_requirements( self.filename, options=PipOption(self.config) ).items() } except pip_api.exceptions.PipError as e: raise ReqsError('%s (from -r %s)' % ( e.args[0].split('\n')[0], self.filename, ))
def test_include_invalid_requirement(monkeypatch): requirement_text = "git+https://github.com/bar/foo" files = {"a.txt": [requirement_text + "\n"]} monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt", include_invalid=True) assert set(result) == {requirement_text} assert result[requirement_text].name == requirement_text assert (str(result[requirement_text]) == dedent(""" Invalid requirement: '{requirement_text}' It looks like a path. File '{requirement_text}' does not exist. """.format(requirement_text=requirement_text)).strip())
def test_parse_requirements_multiline(monkeypatch, lines): files = { 'a.txt': lines, 'b.txt': [ 'foo==1.2.3\n', ], } monkeypatch.setattr(pip_api._parse_requirements, '_read_file', files.__getitem__) result = pip_api.parse_requirements('a.txt') assert set(result) == {'foo'} assert str(result['foo']) == 'foo==1.2.3'
def test_parse_requirements_editable(monkeypatch): files = { 'a.txt': ["Django==1.11\n" "-e git+https://github.com/foo/deal.git#egg=deal\n"], } monkeypatch.setattr(pip_api._parse_requirements, '_read_file', files.get) result = pip_api.parse_requirements('a.txt') assert set(result) == {'django', 'deal'} assert str(result['django']) == 'Django==1.11' assert str( result['deal']) == 'deal@ git+https://github.com/foo/deal.git#egg=deal'
def test_parse_requirements_editable(monkeypatch): files = { "a.txt": ["Django==1.11\n" "-e git+https://github.com/foo/deal.git#egg=deal\n"] } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt") assert set(result) == {"django", "deal"} assert str(result["django"]) == "Django==1.11" assert str( result["deal"]) == "deal@ git+https://github.com/foo/deal.git#egg=deal"
def test_parse_requirements_with_environment_markers(monkeypatch): files = { "a.txt": [ "foo==1.2.3 ; python_version <= '2.7'\n", "foo==3.2.1 ; python_version > '2.7'\n", ] } monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get) result = pip_api.parse_requirements("a.txt") # We don't support such old Python versions, so if we've managed to run these tests, we should # have chosen foo==3.2.1 assert set(result) == {"foo"} assert str(result["foo"]) == 'foo==3.2.1; python_version > "2.7"'
def test_parse_requirements_recursive(monkeypatch, flag): files = { 'a.txt': [ '{} b.txt\n'.format(flag), ], 'b.txt': [ 'foo==1.2.3\n', ], } monkeypatch.setattr(pip_api._parse_requirements, '_read_file', files.get) result = pip_api.parse_requirements('a.txt') assert set(result) == {'foo'} assert str(result['foo']) == 'foo==1.2.3'
def check_requirements(): # Only check requirements in dev environments if IS_EXE: return all_req_files = [ "requirements.txt", "requirements-dev.txt", ] if is_windows(): all_req_files.append("requirements-win.txt") installed = pip_api.installed_distributions() missing_req_files = set() for req_file in all_req_files: requirements = pip_api.parse_requirements(req_file) for req in requirements.values(): if req.name not in installed: missing_req_files.add(req_file) logger.warning("Missing required package '%s'", req.name) continue installed_ver = installed[req.name].version if req.specifier.contains(installed_ver): continue missing_req_files.add(req_file) logger.warning( "Installed version of '%s' is %s, doesen't meet %s", req.name, installed_ver, req.specifier, ) if not missing_req_files: return r_args = " ".join([f"-r {r}" for r in missing_req_files]) logger.warning( "Some requirements aren't met. Run pip install --upgrade %s", r_args, )
def pipdownload(packages, index_url, requirement_file, dest_dir, whl_suffixes, platform_tags, python_versions, quiet, no_source, show_config, show_urls): """ pip-download is a tool which can be used to download python projects and their dependencies listed on pypi's `download files` page. It can be used to download Python packages across system platforms and Python versions. """ if show_config: if not Path(settings.SETTINGS_FILE).exists(): Path(settings.SETTINGS_FILE).parent.mkdir(parents=True, exist_ok=True) # Path(SETTINGS_FILE).touch() with open(settings.SETTINGS_FILE, "w", encoding="utf8") as f: json.dump({}, f) click.echo(f"The config file is {settings.SETTINGS_FILE}.") sys.exit(0) if Path(settings.SETTINGS_FILE).exists(): with open(settings.SETTINGS_FILE, "r") as f: try: settings_dict = json.loads(f.read(), object_pairs_hook=OrderedDict) except json.decoder.JSONDecodeError: logger.error( f"The config file {settings.SETTINGS_FILE} is not correct, it should be a json object." ) sys.exit(-2) if not python_versions: python_versions = settings_dict.get("python-versions", None) if python_versions: click.echo(f"Using `python-versions` in config file.") if not (platform_tags or whl_suffixes): platform_tags = settings_dict.get("platform-tags", None) if platform_tags: click.echo(f"Using `platform-tags` in config file.") tz = get_localzone() if tz.zone in ["Asia/Shanghai", "Asia/Chongqing"]: index_url = "https://mirrors.aliyun.com/pypi/simple/" if whl_suffixes: warnings.warn( "Option '-s/--suffix' has been deprecated. Please use '-p/--platform-tag' instead." ) platform_tags = whl_suffixes if quiet: logger.setLevel(logging.ERROR) download = quiet_download else: download = normal_download url_list = [] if not dest_dir: dest_dir = os.getcwd() else: if not os.path.exists(dest_dir): os.makedirs(dest_dir) # dest_dir = os.path.abspath(dest_dir) if requirement_file: packages_extra_dict = pip_api.parse_requirements(requirement_file) packages_extra = {str(value) for value in packages_extra_dict.values()} else: packages_extra = set() for package in itertools.chain(packages_extra, packages): with TempDirectory(delete=True) as directory: logger.info( "We are using pip download command to download package %s" % package) logger.info("-" * 50) try: command = [ sys.executable, "-m", "pip", "download", "-i", index_url, "--dest", directory.path, package, ] if quiet: command.extend(["--progress-bar", "off", "-qqq"]) subprocess.check_call(command) except subprocess.CalledProcessError as e: logger.error( "Sorry, we can not use pip download to download the package %s," " and Exception is below" % package) logger.error(e) raise file_names = os.listdir(directory.path) for file_name in file_names: python_package = resolve_package_file(file_name) url_list.append(python_package) if python_package.name is None: logger.warning( "Can not resolve a package's name and version from a downloaded package. You shuold " "create an issue maybe.") continue url = mkurl_pypi_url(index_url, python_package.name) try: r = session.get(url) for file in get_file_links(r.text, url, python_package): url_list.append(file) if "none-any" in file: download(file, dest_dir) continue if ".tar.gz" in file or ".zip" in file: if not no_source: download(file, dest_dir) continue eligible = True if platform_tags: for tag in platform_tags: if tag in file: eligible = True break else: eligible = False if not eligible: continue if python_versions: for version in python_versions: if version in file: eligible = True break else: eligible = False if eligible: download(file, dest_dir) except ConnectionError as e: logger.error( "Can not get information about package %s, and the Exception is below.", python_package.name, ) logger.error(e) raise logger.info("All packages have been downloaded successfully!") if show_urls: logger.setLevel(logging.INFO) logger.error("List of files downloaded :") for entry in url_list: logger.info(entry) return url_list
def pipdownload(packages, index_url, requirement_file, dest_dir, whl_suffixes, python_versions, quiet): """ pip-download is a tool which can be used to download python projects and their dependencies listed on pypi's `download files` page. """ tz = get_localzone() if tz.zone in ['Asia/Shanghai', 'Asia/Chongqing']: index_url = 'https://pypi.tuna.tsinghua.edu.cn/simple' if quiet: logger.setLevel(logging.ERROR) download = quiet_download else: download = normal_download if not dest_dir: dest_dir = os.getcwd() else: if not os.path.exists(dest_dir): os.makedirs(dest_dir) # dest_dir = os.path.abspath(dest_dir) if requirement_file: packages_extra_dict = pip_api.parse_requirements(requirement_file) packages_extra = {str(value) for value in packages_extra_dict.values()} else: packages_extra = set() for package in itertools.chain(packages_extra, packages): with TempDirectory(delete=True) as directory: logger.info( 'We are using pip download command to download package %s' % package) logger.info('-' * 50) try: command = [ sys.executable, '-m', 'pip', 'download', '-i', index_url, '--dest', directory.path, package ] if quiet: command.extend(['--progress-bar', 'off', '-qqq']) subprocess.check_call(command) except subprocess.CalledProcessError as e: logger.error( 'Sorry, we can not use pip download to download the package %s,' ' and Exception is below' % package) logger.error(e) raise file_names = os.listdir(directory.path) # if not exact: # logger.info('-' * 50) # logger.info('At First, we copy these files to %s' % dest_dir) # for file_name in file_names: # shutil.copy(os.path.join(directory.path, file_name), dest_dir) # logger.info('All file have been copied!') for python_package in resolve_package_files(file_names): url = mkurl_pypi_url(index_url, python_package.name) try: r = session.get(url) for file in get_file_links(r.text, index_url, python_package): if 'tar.gz' in file: download(file, dest_dir) continue if 'none-any' in file: download(file, dest_dir) continue if 'zip' in file: download(file, dest_dir) continue eligible = True if whl_suffixes: for suffix in whl_suffixes: if suffix in file: eligible = True break else: eligible = False if not eligible: continue if python_versions: for version in python_versions: if version in file: eligible = True break else: eligible = False if eligible: download(file, dest_dir) except ConnectionError as e: logger.error( 'Can not get information about package %s, and the Exception is below.', python_package.name) logger.error(e) raise logger.info('All packages have been downloaded successfully!')
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.import sys import sys from pip_api import parse_requirements left, right = sys.argv[1:3] left_reqs = parse_requirements(left).keys() right_reqs = parse_requirements(right).keys() extra_in_left = left_reqs - right_reqs extra_in_right = right_reqs - left_reqs if extra_in_left: for dep in sorted(extra_in_left): print("- {}".format(dep)) if extra_in_right: for dep in sorted(extra_in_right): print("+ {}".format(dep)) if extra_in_left or extra_in_right: sys.exit(1)