def test_parse_requirements_vcs(monkeypatch):
requirement_text = "git+https://github.com/bar/foo"
files = {"a.txt": [requirement_text + "\n"]}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
with pytest.raises(PipError):
pip_api.parse_requirements("a.txt")
def test_parse_requirements_missing_hashes_late(monkeypatch, strict_hashes):
files = {
"a.txt": [
"foo==1.2.3\n",
"bar==1.2.3\n",
"baz==1.2.3 --hash=sha256:862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e\n",
]
}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
if strict_hashes:
with pytest.raises(
PipError,
match=r"Missing hashes for requirement in a\.txt, line 1"):
pip_api.parse_requirements("a.txt", strict_hashes=strict_hashes)
else:
result = pip_api.parse_requirements("a.txt",
strict_hashes=strict_hashes)
assert result["foo"].hashes == {}
assert result["bar"].hashes == {}
assert result["baz"].hashes == {
"sha256": [
"862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e"
],
}
def test_parse_requirements_invalid_hash_kind(monkeypatch):
files = {
"a.txt": ["foo==1.2.3 --hash=md5:0d5a28f01dccb5a549c31016883f59c2"]
}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
with pytest.raises(PipError, match=r"Invalid --hash kind"):
pip_api.parse_requirements("a.txt")
def test_parse_requirements_double_raises(monkeypatch):
files = {"a.txt": ["foo==1.2.3\n", "foo==3.2.1\n"]}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
with pytest.raises(pip_api.exceptions.PipError) as e:
pip_api.parse_requirements("a.txt")
assert e.value.args == (
"Double requirement given: foo==3.2.1 (already in foo==1.2.3, name='foo')",
)
def test_parse_requirements_with_invalid_wheel_filename(monkeypatch):
INVALID_WHEEL_NAME = "pip-1.3.1-invalid-format.whl"
files = {
"a.txt": ["https://github.com/pypa/pip/archive/" + INVALID_WHEEL_NAME],
}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
with pytest.raises(PipError,
match=r"Invalid wheel name: " + INVALID_WHEEL_NAME):
pip_api.parse_requirements("a.txt")
def test_parse_requirements_with_missing_egg_suffix(monkeypatch):
# Without a package name, an `#egg=foo` suffix is required to know the package name
files = {
"a.txt": [PEP508_PIP_EXAMPLE_URL],
}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
with pytest.raises(PipError,
match=r"Missing egg fragment in URL: " +
PEP508_PIP_EXAMPLE_URL):
pip_api.parse_requirements("a.txt")
def test_parse_requirements_double_raises(monkeypatch):
files = {
'a.txt': [
'foo==1.2.3\n',
'foo==3.2.1\n',
],
}
monkeypatch.setattr(pip_api._parse_requirements, '_read_file', files.get)
with pytest.raises(pip_api.exceptions.PipError) as e:
pip_api.parse_requirements('a.txt')
assert e.value.args == (
"Double requirement given: foo==3.2.1 (already in foo==1.2.3, name='foo')", # noqa
)
def test_parse_requirements_missing_all_hashes_strict(monkeypatch):
files = {
"a.txt": [
"foo==1.2.3\n",
"bar==1.2.3\n",
"baz==1.2.3\n",
]
}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
with pytest.raises(
PipError,
match=r"Missing hashes for requirement in a\.txt, line 1"):
pip_api.parse_requirements("a.txt", strict_hashes=True)
def test_parse_requirements_hashes(monkeypatch):
files = {
"a.txt": [
"foo==1.2.3 "
"--hash=sha256:862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e "
"--hash=sha256:0cfea7e5a53d5a256b4e8609c8a1812ad9af5c611432ec9dccbb4d79dc6a336e "
"--hash=sha384:673546e6c3236a36e5db5f1bc9d2cb5f3f974d3d4e9031f405b1dc7874575e2ad91436d02edf8237a889ab1cecb35d56 "
"--hash=sha512:3b149832490a704091abed6a9bd40ef7f4176b279263d4cbbb440b067ced99cadc006c03bc47488755351022fb49f2f10edfec110f027039bda703d407135c47"
]
}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("a.txt")
assert set(result) == {"foo"}
assert result["foo"].hashes == {
"sha256": [
"862db587c4257f71293cf07cafc521961712c088a52981f3d81be056eaabc95e",
"0cfea7e5a53d5a256b4e8609c8a1812ad9af5c611432ec9dccbb4d79dc6a336e",
],
"sha384": [
"673546e6c3236a36e5db5f1bc9d2cb5f3f974d3d4e9031f405b1dc7874575e2ad91436d02edf8237a889ab1cecb35d56"
],
"sha512": [
"3b149832490a704091abed6a9bd40ef7f4176b279263d4cbbb440b067ced99cadc006c03bc47488755351022fb49f2f10edfec110f027039bda703d407135c47"
],
}
def test_parse_requirements_recursive(monkeypatch, flag):
files = {"a.txt": ["{} b.txt\n".format(flag)], "b.txt": ["foo==1.2.3\n"]}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("a.txt")
assert set(result) == {"foo"}
assert str(result["foo"]) == "foo==1.2.3"
def test_parse_requirements(monkeypatch):
files = {"a.txt": ["foo==1.2.3\n"]}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("a.txt")
assert set(result) == {"foo"}
assert str(result["foo"]) == "foo==1.2.3"
def test_parse_requirements_editable_file(monkeypatch):
files = {"a.txt": ["Django==1.11\n" "-e .\n"]}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("a.txt")
assert set(result) == {"django", "pip-api"}
assert str(result["django"]) == "Django==1.11"
assert str(result["pip-api"]).startswith("pip-api@ file:///")
def test_parse_requirements_multiline(monkeypatch, lines):
files = {"a.txt": lines, "b.txt": ["foo==1.2.3\n"]}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file",
files.__getitem__)
result = pip_api.parse_requirements("a.txt")
assert set(result) == {"foo"}
assert str(result["foo"]) == "foo==1.2.3"
def _get_names_cached(cls, path):
results = []
with chdir(os.path.dirname(path)):
requirements = parse_requirements(path)
for req in requirements.values():
if req.name:
results.append(req.name)
return results
def _get_names_cached(cls, path):
results = []
with chdir(os.path.dirname(path)):
requirements = parse_requirements(path)
for req in requirements.values():
if req.name:
results.append(req.name)
return results
def get_requirements(self):
try:
return {
packaging.utils.canonicalize_name(name): req
for name, req in pip_api.parse_requirements(
self.filename, options=PipOption(self.config)).items()
}
except pip_api.exceptions.PipError as e:
raise ReqsError("%s (from -r %s)" %
(e.args[0].split("\n")[0], self.filename))
def test_parse_requirements_with_relative_references(monkeypatch):
files = {
"reqs/base.txt": ["django==1.11\n"],
"reqs/test.txt": ["-r base.txt\n"],
"reqs/dev.txt": ["-r base.txt\n"
"-r test.txt\n"],
}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("reqs/dev.txt")
assert set(result) == {"django"}
def test_parse_requirements_PEP508(monkeypatch, line, result_set, url, string,
spec):
files = {"a.txt": [line]}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("a.txt")
assert set(result) == result_set
assert result["pip"].url == url
assert str(result["pip"]) == string
assert result["pip"].specifier == spec
def test_include_invalid_requirement(monkeypatch):
requirement_text = "git+https://github.com/bar/foo"
files = {"a.txt": [requirement_text + "\n"]}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("a.txt", include_invalid=True)
assert set(result) == {requirement_text}
assert result[requirement_text].name == requirement_text
assert (str(result[requirement_text]) ==
f"Missing egg fragment in URL: {requirement_text}")
def test_parse_requirements_with_index_url(monkeypatch, flag):
files = {
"a.txt":
["{} https://example.com/pypi/simple\n".format(flag), "foo==1.2.3\n"]
}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("a.txt")
assert set(result) == {"foo"}
assert str(result["foo"]) == "foo==1.2.3"
def test_parse_requirements(monkeypatch):
files = {
'a.txt': [
'foo==1.2.3\n',
],
}
monkeypatch.setattr(pip_api._parse_requirements, '_read_file', files.get)
result = pip_api.parse_requirements('a.txt')
assert set(result) == {'foo'}
assert str(result['foo']) == 'foo==1.2.3'
def get_requirements(self):
try:
return {
packaging.utils.canonicalize_name(name): req
for name, req in pip_api.parse_requirements(
self.filename, options=PipOption(self.config)
).items()
}
except pip_api.exceptions.PipError as e:
raise ReqsError('%s (from -r %s)' % (
e.args[0].split('\n')[0],
self.filename,
))
def test_include_invalid_requirement(monkeypatch):
requirement_text = "git+https://github.com/bar/foo"
files = {"a.txt": [requirement_text + "\n"]}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("a.txt", include_invalid=True)
assert set(result) == {requirement_text}
assert result[requirement_text].name == requirement_text
assert (str(result[requirement_text]) == dedent("""
Invalid requirement: '{requirement_text}'
It looks like a path. File '{requirement_text}' does not exist.
""".format(requirement_text=requirement_text)).strip())
def test_parse_requirements_multiline(monkeypatch, lines):
files = {
'a.txt': lines,
'b.txt': [
'foo==1.2.3\n',
],
}
monkeypatch.setattr(pip_api._parse_requirements, '_read_file',
files.__getitem__)
result = pip_api.parse_requirements('a.txt')
assert set(result) == {'foo'}
assert str(result['foo']) == 'foo==1.2.3'
def test_parse_requirements_editable(monkeypatch):
files = {
'a.txt':
["Django==1.11\n"
"-e git+https://github.com/foo/deal.git#egg=deal\n"],
}
monkeypatch.setattr(pip_api._parse_requirements, '_read_file', files.get)
result = pip_api.parse_requirements('a.txt')
assert set(result) == {'django', 'deal'}
assert str(result['django']) == 'Django==1.11'
assert str(
result['deal']) == 'deal@ git+https://github.com/foo/deal.git#egg=deal'
def test_parse_requirements_editable(monkeypatch):
files = {
"a.txt":
["Django==1.11\n"
"-e git+https://github.com/foo/deal.git#egg=deal\n"]
}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("a.txt")
assert set(result) == {"django", "deal"}
assert str(result["django"]) == "Django==1.11"
assert str(
result["deal"]) == "deal@ git+https://github.com/foo/deal.git#egg=deal"
def test_parse_requirements_with_environment_markers(monkeypatch):
files = {
"a.txt": [
"foo==1.2.3 ; python_version <= '2.7'\n",
"foo==3.2.1 ; python_version > '2.7'\n",
]
}
monkeypatch.setattr(pip_api._parse_requirements, "_read_file", files.get)
result = pip_api.parse_requirements("a.txt")
# We don't support such old Python versions, so if we've managed to run these tests, we should
# have chosen foo==3.2.1
assert set(result) == {"foo"}
assert str(result["foo"]) == 'foo==3.2.1; python_version > "2.7"'
def test_parse_requirements_recursive(monkeypatch, flag):
files = {
'a.txt': [
'{} b.txt\n'.format(flag),
],
'b.txt': [
'foo==1.2.3\n',
],
}
monkeypatch.setattr(pip_api._parse_requirements, '_read_file', files.get)
result = pip_api.parse_requirements('a.txt')
assert set(result) == {'foo'}
assert str(result['foo']) == 'foo==1.2.3'
def check_requirements():
# Only check requirements in dev environments
if IS_EXE:
return
all_req_files = [
"requirements.txt",
"requirements-dev.txt",
]
if is_windows():
all_req_files.append("requirements-win.txt")
installed = pip_api.installed_distributions()
missing_req_files = set()
for req_file in all_req_files:
requirements = pip_api.parse_requirements(req_file)
for req in requirements.values():
if req.name not in installed:
missing_req_files.add(req_file)
logger.warning("Missing required package '%s'", req.name)
continue
installed_ver = installed[req.name].version
if req.specifier.contains(installed_ver):
continue
missing_req_files.add(req_file)
logger.warning(
"Installed version of '%s' is %s, doesen't meet %s",
req.name,
installed_ver,
req.specifier,
)
if not missing_req_files:
return
r_args = " ".join([f"-r {r}" for r in missing_req_files])
logger.warning(
"Some requirements aren't met. Run pip install --upgrade %s",
r_args,
)
def pipdownload(packages, index_url, requirement_file, dest_dir, whl_suffixes,
platform_tags, python_versions, quiet, no_source, show_config,
show_urls):
"""
pip-download is a tool which can be used to download python projects and their dependencies listed on
pypi's `download files` page. It can be used to download Python packages across system platforms and
Python versions.
"""
if show_config:
if not Path(settings.SETTINGS_FILE).exists():
Path(settings.SETTINGS_FILE).parent.mkdir(parents=True,
exist_ok=True)
# Path(SETTINGS_FILE).touch()
with open(settings.SETTINGS_FILE, "w", encoding="utf8") as f:
json.dump({}, f)
click.echo(f"The config file is {settings.SETTINGS_FILE}.")
sys.exit(0)
if Path(settings.SETTINGS_FILE).exists():
with open(settings.SETTINGS_FILE, "r") as f:
try:
settings_dict = json.loads(f.read(),
object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError:
logger.error(
f"The config file {settings.SETTINGS_FILE} is not correct, it should be a json object."
)
sys.exit(-2)
if not python_versions:
python_versions = settings_dict.get("python-versions", None)
if python_versions:
click.echo(f"Using `python-versions` in config file.")
if not (platform_tags or whl_suffixes):
platform_tags = settings_dict.get("platform-tags", None)
if platform_tags:
click.echo(f"Using `platform-tags` in config file.")
tz = get_localzone()
if tz.zone in ["Asia/Shanghai", "Asia/Chongqing"]:
index_url = "https://mirrors.aliyun.com/pypi/simple/"
if whl_suffixes:
warnings.warn(
"Option '-s/--suffix' has been deprecated. Please use '-p/--platform-tag' instead."
)
platform_tags = whl_suffixes
if quiet:
logger.setLevel(logging.ERROR)
download = quiet_download
else:
download = normal_download
url_list = []
if not dest_dir:
dest_dir = os.getcwd()
else:
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
# dest_dir = os.path.abspath(dest_dir)
if requirement_file:
packages_extra_dict = pip_api.parse_requirements(requirement_file)
packages_extra = {str(value) for value in packages_extra_dict.values()}
else:
packages_extra = set()
for package in itertools.chain(packages_extra, packages):
with TempDirectory(delete=True) as directory:
logger.info(
"We are using pip download command to download package %s" %
package)
logger.info("-" * 50)
try:
command = [
sys.executable,
"-m",
"pip",
"download",
"-i",
index_url,
"--dest",
directory.path,
package,
]
if quiet:
command.extend(["--progress-bar", "off", "-qqq"])
subprocess.check_call(command)
except subprocess.CalledProcessError as e:
logger.error(
"Sorry, we can not use pip download to download the package %s,"
" and Exception is below" % package)
logger.error(e)
raise
file_names = os.listdir(directory.path)
for file_name in file_names:
python_package = resolve_package_file(file_name)
url_list.append(python_package)
if python_package.name is None:
logger.warning(
"Can not resolve a package's name and version from a downloaded package. You shuold "
"create an issue maybe.")
continue
url = mkurl_pypi_url(index_url, python_package.name)
try:
r = session.get(url)
for file in get_file_links(r.text, url, python_package):
url_list.append(file)
if "none-any" in file:
download(file, dest_dir)
continue
if ".tar.gz" in file or ".zip" in file:
if not no_source:
download(file, dest_dir)
continue
eligible = True
if platform_tags:
for tag in platform_tags:
if tag in file:
eligible = True
break
else:
eligible = False
if not eligible:
continue
if python_versions:
for version in python_versions:
if version in file:
eligible = True
break
else:
eligible = False
if eligible:
download(file, dest_dir)
except ConnectionError as e:
logger.error(
"Can not get information about package %s, and the Exception is below.",
python_package.name,
)
logger.error(e)
raise
logger.info("All packages have been downloaded successfully!")
if show_urls:
logger.setLevel(logging.INFO)
logger.error("List of files downloaded :")
for entry in url_list:
logger.info(entry)
return url_list
def pipdownload(packages, index_url, requirement_file, dest_dir, whl_suffixes,
python_versions, quiet):
"""
pip-download is a tool which can be used to download python projects and their dependencies listed on
pypi's `download files` page.
"""
tz = get_localzone()
if tz.zone in ['Asia/Shanghai', 'Asia/Chongqing']:
index_url = 'https://pypi.tuna.tsinghua.edu.cn/simple'
if quiet:
logger.setLevel(logging.ERROR)
download = quiet_download
else:
download = normal_download
if not dest_dir:
dest_dir = os.getcwd()
else:
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
# dest_dir = os.path.abspath(dest_dir)
if requirement_file:
packages_extra_dict = pip_api.parse_requirements(requirement_file)
packages_extra = {str(value) for value in packages_extra_dict.values()}
else:
packages_extra = set()
for package in itertools.chain(packages_extra, packages):
with TempDirectory(delete=True) as directory:
logger.info(
'We are using pip download command to download package %s' %
package)
logger.info('-' * 50)
try:
command = [
sys.executable, '-m', 'pip', 'download', '-i', index_url,
'--dest', directory.path, package
]
if quiet:
command.extend(['--progress-bar', 'off', '-qqq'])
subprocess.check_call(command)
except subprocess.CalledProcessError as e:
logger.error(
'Sorry, we can not use pip download to download the package %s,'
' and Exception is below' % package)
logger.error(e)
raise
file_names = os.listdir(directory.path)
# if not exact:
# logger.info('-' * 50)
# logger.info('At First, we copy these files to %s' % dest_dir)
# for file_name in file_names:
# shutil.copy(os.path.join(directory.path, file_name), dest_dir)
# logger.info('All file have been copied!')
for python_package in resolve_package_files(file_names):
url = mkurl_pypi_url(index_url, python_package.name)
try:
r = session.get(url)
for file in get_file_links(r.text, index_url,
python_package):
if 'tar.gz' in file:
download(file, dest_dir)
continue
if 'none-any' in file:
download(file, dest_dir)
continue
if 'zip' in file:
download(file, dest_dir)
continue
eligible = True
if whl_suffixes:
for suffix in whl_suffixes:
if suffix in file:
eligible = True
break
else:
eligible = False
if not eligible:
continue
if python_versions:
for version in python_versions:
if version in file:
eligible = True
break
else:
eligible = False
if eligible:
download(file, dest_dir)
except ConnectionError as e:
logger.error(
'Can not get information about package %s, and the Exception is below.',
python_package.name)
logger.error(e)
raise
logger.info('All packages have been downloaded successfully!')
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.import sys
import sys
from pip_api import parse_requirements
left, right = sys.argv[1:3]
left_reqs = parse_requirements(left).keys()
right_reqs = parse_requirements(right).keys()
extra_in_left = left_reqs - right_reqs
extra_in_right = right_reqs - left_reqs
if extra_in_left:
for dep in sorted(extra_in_left):
print("- {}".format(dep))
if extra_in_right:
for dep in sorted(extra_in_right):
print("+ {}".format(dep))
if extra_in_left or extra_in_right:
sys.exit(1)