def populate_requirement_set(requirement_set, args, options, finder,
session, name, wheel_cache):
"""
Marshal cmd line args into a requirement set.
"""
# NOTE: As a side-effect, options.require_hashes and
# requirement_set.require_hashes may be updated
for filename in options.constraints:
for req_to_add in parse_requirements(
filename,
constraint=True,
finder=finder,
options=options,
session=session,
wheel_cache=wheel_cache,
):
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
for req in args:
req_to_add = install_req_from_line(req,
None,
isolated=options.isolated_mode,
wheel_cache=wheel_cache)
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
for req in options.editables:
req_to_add = install_req_from_editable(
req, isolated=options.isolated_mode, wheel_cache=wheel_cache)
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
for filename in options.requirements:
for req_to_add in parse_requirements(
filename,
finder=finder,
options=options,
session=session,
wheel_cache=wheel_cache,
):
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
# If --require-hashes was a line in a requirements file, tell
# RequirementSet about it:
requirement_set.require_hashes = options.require_hashes
if not (args or options.editables or options.requirements):
opts = {"name": name}
if options.find_links:
raise CommandError(
"You must give at least one requirement to %(name)s "
'(maybe you meant "pip %(name)s %(links)s"?)' %
dict(opts, links=" ".join(options.find_links)))
else:
raise CommandError(
"You must give at least one requirement to %(name)s "
'(see "pip help %(name)s")' % opts)
def python_validator(data):
"""Very simple Python requirements.txt validator."""
requirements_txt = StringIO()
requirements_txt.write(data)
try:
parse_requirements(requirements_txt, session="requirements.txt")
except RequirementsFileParseError:
return False
return True
def populate_requirement_set(requirement_set, args, options, finder,
session, name, wheel_cache):
"""
Marshal cmd line args into a requirement set.
"""
# NOTE: As a side-effect, options.require_hashes and
# requirement_set.require_hashes may be updated
for filename in options.constraints:
for req_to_add in parse_requirements(
filename,
constraint=True, finder=finder, options=options,
session=session, wheel_cache=wheel_cache):
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
for req in args:
req_to_add = InstallRequirement.from_line(
req, None, isolated=options.isolated_mode,
wheel_cache=wheel_cache
)
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
for req in options.editables:
req_to_add = InstallRequirement.from_editable(
req,
isolated=options.isolated_mode,
wheel_cache=wheel_cache
)
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
for filename in options.requirements:
for req_to_add in parse_requirements(
filename,
finder=finder, options=options, session=session,
wheel_cache=wheel_cache):
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
# If --require-hashes was a line in a requirements file, tell
# RequirementSet about it:
requirement_set.require_hashes = options.require_hashes
if not (args or options.editables or options.requirements):
opts = {'name': name}
if options.find_links:
raise CommandError(
'You must give at least one requirement to %(name)s '
'(maybe you meant "pip %(name)s %(links)s"?)' %
dict(opts, links=' '.join(options.find_links)))
else:
raise CommandError(
'You must give at least one requirement to %(name)s '
'(see "pip help %(name)s")' % opts)
def test_req_file_no_finder(self, tmpdir):
"""
Test parsing a requirements file without a finder
"""
with open(tmpdir.joinpath("req.txt"), "w") as fp:
fp.write("""
--find-links https://example.com/
--index-url https://example.com/
--extra-index-url https://two.example.com/
--no-use-wheel
--no-index
""")
parse_requirements(tmpdir.joinpath("req.txt"), session=PipSession())
def test_req_file_no_finder(self, tmpdir):
"""
Test parsing a requirements file without a finder
"""
with open(tmpdir.join("req.txt"), "w") as fp:
fp.write("""
--find-links https://example.com/
--index-url https://example.com/
--extra-index-url https://two.example.com/
--no-use-wheel
--no-index
""")
parse_requirements(tmpdir.join("req.txt"), session=PipSession())
def test_install_requirements_with_options(self, tmpdir, finder, session,
options):
global_option = '--dry-run'
install_option = '--prefix=/opt'
content = '''
--only-binary :all:
INITools==2.0 --global-option="{global_option}" \
--install-option "{install_option}"
'''.format(global_option=global_option, install_option=install_option)
with requirements_file(content, tmpdir) as reqs_file:
req = next(
parse_requirements(reqs_file.abspath,
finder=finder,
options=options,
session=session))
req.source_dir = os.curdir
with patch.object(subprocess, 'Popen') as popen:
popen.return_value.stdout.readline.return_value = b""
try:
req.install([])
except Exception:
pass
last_call = popen.call_args_list[-1]
args = last_call[0][0]
assert (0 < args.index(global_option) < args.index('install') <
args.index(install_option))
assert options.format_control.no_binary == {':all:'}
assert options.format_control.only_binary == set()
def merge_requirements(files):
requirements = defaultdict(lambda: Requirement())
links = set()
for filename in files:
for requirement in parse_requirements(filename, session=PipSession()):
if not hasattr(requirement.req, 'name'):
links.add(requirement.link.url)
break
name = requirement.req.name
specifiers = requirement.req.specifier
extras = requirement.req.extras
requirements[name].extras |= set(extras)
requirements[name].specifiers |= set(specifiers)
result = []
for key, value in requirements.items():
if not value.extras:
result.append("%s %s" %
(key, ",".join(map(str, value.specifiers))))
else:
result.append("%s [%s] %s" % (key, ",".join(map(
str, value.extras)), ",".join(map(str, value.specifiers))))
for link in links:
result.append(link)
return "\n".join(result)
def test_req_file_parse_no_only_binary(self, data, finder):
list(
parse_requirements(data.reqfiles.join("supported_options2.txt"),
finder,
session=PipSession()))
expected = FormatControl({'fred'}, {'wilma'})
assert finder.format_control == expected
def test_install_requirements_with_options(self, tmpdir, finder, session,
options):
global_option = '--dry-run'
install_option = '--prefix=/opt'
content = '''
--only-binary :all:
INITools==2.0 --global-option="{global_option}" \
--install-option "{install_option}"
'''.format(global_option=global_option, install_option=install_option)
with requirements_file(content, tmpdir) as reqs_file:
req = next(parse_requirements(reqs_file.abspath,
finder=finder,
options=options,
session=session))
req.source_dir = os.curdir
with patch.object(subprocess, 'Popen') as popen:
popen.return_value.stdout.readline.return_value = b""
try:
req.install([])
except:
pass
last_call = popen.call_args_list[-1]
args = last_call[0][0]
assert (
0 < args.index(global_option) < args.index('install') <
args.index(install_option)
)
assert options.format_control.no_binary == {':all:'}
assert options.format_control.only_binary == set()
def test_absolute_http_nested_req_file_in_local(
self, session, monkeypatch, tmpdir
):
"""
Test a nested req file url in a local req file
"""
req_name = 'hello'
req_file = tmpdir / 'req_file.txt'
nested_req_file = 'http://me.com/me/req_file.txt'
def get_file_content(filename, *args, **kwargs):
if filename == str(req_file):
return None, '-r {}'.format(nested_req_file)
elif filename == nested_req_file:
return None, req_name
assert False, 'Unexpected file requested {}'.format(filename)
monkeypatch.setattr(
pip._internal.req.req_file, 'get_file_content', get_file_content
)
result = list(parse_requirements(req_file, session=session))
assert len(result) == 1
assert result[0].name == req_name
assert not result[0].constraint
def test_expand_existing_env_variables(self, tmpdir, finder):
template = (
'https://%s:[email protected]/user/%s/archive/master.zip')
env_vars = (
('GITHUB_TOKEN', 'notarealtoken'),
('DO_12_FACTOR', 'awwyeah'),
)
with open(tmpdir.joinpath('req1.txt'), 'w') as fp:
fp.write(template % tuple(['${%s}' % k for k, _ in env_vars]))
with patch('pip._internal.req.req_file.os.getenv') as getenv:
getenv.side_effect = lambda n: dict(env_vars)[n]
reqs = list(
parse_requirements(tmpdir.joinpath('req1.txt'),
finder=finder,
session=PipSession()))
assert len(reqs) == 1, \
'parsing requirement file with env variable failed'
expected_url = template % tuple([v for _, v in env_vars])
assert reqs[0].link.url == expected_url, \
'variable expansion in req file failed'
def test_expand_existing_env_variables(self, tmpdir, finder):
template = (
'https://%s:[email protected]/user/%s/archive/master.zip'
)
env_vars = (
('GITHUB_TOKEN', 'notarealtoken'),
('DO_12_FACTOR', 'awwyeah'),
)
with open(tmpdir.join('req1.txt'), 'w') as fp:
fp.write(template % tuple(['${%s}' % k for k, _ in env_vars]))
with patch('pip._internal.req.req_file.os.getenv') as getenv:
getenv.side_effect = lambda n: dict(env_vars)[n]
reqs = list(parse_requirements(
tmpdir.join('req1.txt'),
finder=finder,
session=PipSession()
))
assert len(reqs) == 1, \
'parsing requirement file with env variable failed'
expected_url = template % tuple([v for _, v in env_vars])
assert reqs[0].link.url == expected_url, \
'variable expansion in req file failed'
def process_line(
line,
filename,
line_number,
finder=None,
options=None,
session=None,
constraint=False,
):
if session is None:
session = PipSession()
prefix = '\n' * (line_number - 1)
path = tmpdir.joinpath(filename)
path.parent.mkdir(exist_ok=True)
path.write_text(prefix + line)
monkeypatch.chdir(str(tmpdir))
return list(
parse_requirements(
filename,
finder=finder,
options=options,
session=session,
constraint=constraint,
))
def test_req_file_parse_no_only_binary(self, data, finder):
list(parse_requirements(
data.reqfiles.join("supported_options2.txt"), finder,
session=PipSession()))
expected = pip._internal.index.FormatControl(
{'fred'}, {'wilma'})
assert finder.format_control == expected
def find_required_modules(options, requirements_filename: str):
explicit = set()
for requirement in parse_requirements(requirements_filename,
session=PipSession()):
try:
requirement_name = requirement.name
# The type of "requirement" changed between pip versions.
# We exclude the "except" from coverage so that on any pip version we
# can report 100% coverage.
except AttributeError: # pragma: no cover
from pip._internal.req.constructors import install_req_from_line
requirement_name = install_req_from_line(
requirement.requirement,
).name
if options.ignore_reqs(requirement):
log.debug('ignoring requirement: %s', requirement_name)
continue
if options.skip_incompatible:
requirement_string = requirement.requirement
if not has_compatible_markers(requirement_string):
log.debug('ignoring requirement (incompatible environment '
'marker): %s', requirement_string)
continue
log.debug('found requirement: %s', requirement_name)
explicit.add(canonicalize_name(requirement_name))
return explicit
def test_join_lines(self, tmpdir, finder):
with open(tmpdir.joinpath("req1.txt"), "w") as fp:
fp.write("--extra-index-url url1 \\\n--extra-index-url url2")
list(parse_requirements(tmpdir.joinpath("req1.txt"), finder=finder,
session=PipSession()))
assert finder.index_urls == ['url1', 'url2']
def test_join_lines(self, tmpdir, finder):
with open(tmpdir.join("req1.txt"), "w") as fp:
fp.write("--extra-index-url url1 \\\n--extra-index-url url2")
list(parse_requirements(tmpdir.join("req1.txt"), finder=finder,
session=PipSession()))
assert finder.index_urls == ['url1', 'url2']
def test_req_file_parse_no_only_binary(self, data, finder):
list(
parse_requirements(data.reqfiles.join("supported_options2.txt"),
finder,
session=PipSession()))
expected = pip._internal.index.FormatControl(set(['fred']),
set(['wilma']))
assert finder.format_control == expected
def handle(self, *args, **options):
self.style = color_style()
self.options = options
if options["requirements"]:
req_files = options["requirements"]
elif os.path.exists("requirements.txt"):
req_files = ["requirements.txt"]
elif os.path.exists("requirements"):
req_files = [
"requirements/{0}".format(f)
for f in os.listdir("requirements")
if os.path.isfile(os.path.join("requirements", f))
and f.lower().endswith(".txt")
]
elif os.path.exists("requirements-dev.txt"):
req_files = ["requirements-dev.txt"]
elif os.path.exists("requirements-prod.txt"):
req_files = ["requirements-prod.txt"]
else:
raise CommandError("Requirements file(s) not found")
self.reqs = {}
with PipSession() as session:
for filename in req_files:
for req in parse_requirements(filename, session=session):
if not isinstance(req, InstallRequirement):
req = install_req_from_line(req.requirement)
name = req.name if req.name else req.link.filename
# url attribute changed to link in pip version 6.1.0 and above
if LooseVersion(pip.__version__) > LooseVersion('6.0.8'):
self.reqs[name] = {
"pip_req": req,
"url": req.link,
}
else:
self.reqs[name] = {
"pip_req": req,
"url": req.url,
}
if options["github_api_token"]:
self.github_api_token = options["github_api_token"]
elif os.environ.get("GITHUB_API_TOKEN"):
self.github_api_token = os.environ.get("GITHUB_API_TOKEN")
else:
self.github_api_token = None # only 50 requests per hour
self.check_pypi()
if HAS_REQUESTS:
self.check_github()
else:
self.stdout.write(
self.style.ERROR(
"Cannot check github urls. The requests library is not installed. ( pip install requests )"
))
self.check_other()
def _parse_python(spec):
"""Parse PyPI specification of a single dependency.
:param spec: str, for example "Django>=1.5,<1.8"
:return: [Django [[('>=', '1.5'), ('<', '1.8')]]]
"""
def _extract_op_version(spec):
# https://www.python.org/dev/peps/pep-0440/#compatible-release
if spec.operator == '~=':
version = _version_split(spec.version)
if len(version) > 1:
# ignore pre-release, post-release or developmental release
while not version[-1].isdigit():
del version[-1]
del version[-1] # will increase the last but one in next line
version[-1] = str(int(version[-1]) + 1)
else:
raise ValueError('%r must not be used with %r' % (spec.operator, spec.version))
return [('>=', spec.version), ('<', '.'.join(version))]
# Trailing .* is permitted per
# https://www.python.org/dev/peps/pep-0440/#version-matching
elif spec.operator == '==' and spec.version.endswith('.*'):
try:
result = check_output(['/usr/bin/semver-ranger', spec.version],
universal_newlines=True).strip()
gte, lt = result.split()
return [('>=', gte.lstrip('>=')), ('<', lt.lstrip('<'))]
except ValueError:
logger.info("couldn't resolve ==%s", spec.version)
return spec.operator, spec.version
# https://www.python.org/dev/peps/pep-0440/#arbitrary-equality
# Use of this operator is heavily discouraged, so just convert it to 'Version matching'
elif spec.operator == '===':
return '==', spec.version
else:
return spec.operator, spec.version
def _get_pip_spec(requirements):
"""There's no `specs` field In Pip 8+, take info from `specifier` field."""
if hasattr(requirements, 'specs'):
return requirements.specs
elif hasattr(requirements, 'specifier'):
specs = [_extract_op_version(spec) for spec in requirements.specifier]
if len(specs) == 0:
specs = [('>=', '0.0.0')]
elif len(specs) > 1:
specs = [specs]
return specs
# create a temporary file and store the spec there since
# `parse_requirements` requires a file
with NamedTemporaryFile(mode='w+', suffix='pysolve') as f:
f.write(spec)
f.flush()
parsed = parse_requirements(f.name, session=f.name)
dependency = [Dependency(x.name, _get_pip_spec(x.req)) for x in parsed].pop()
return dependency
def find_missing_reqs(options, requirements_filename):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug('installed package: %s (at %s)', package['name'],
package['location'])
for package_file in package.get('files', []) or []:
path = os.path.realpath(
os.path.join(package['location'], package_file),
)
installed_files[path] = package['name']
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package['name']
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug('used module: %s (from package %s)', modname,
installed_files[info.filename])
used[used_name].append(info)
else:
log.debug(
'used module: %s (from file %s, assuming stdlib or local)',
modname, info.filename)
# 4. compare with requirements.txt
explicit = set()
for requirement in parse_requirements(
requirements_filename,
session=PipSession(),
):
try:
requirement_name = requirement.name
# The type of "requirement" changed between pip versions.
# We exclude the "except" from coverage so that on any pip version we
# can report 100% coverage.
except AttributeError: # pragma: no cover
from pip._internal.req.constructors import install_req_from_line
requirement_name = install_req_from_line(
requirement.requirement,
).name
log.debug('found requirement: %s', requirement_name)
explicit.add(canonicalize_name(requirement_name))
return [(name, used[name]) for name in used if name not in explicit]
def test_multiple_appending_options(self, tmpdir, finder, options):
with open(tmpdir.joinpath("req1.txt"), "w") as fp:
fp.write("--extra-index-url url1 \n")
fp.write("--extra-index-url url2 ")
list(parse_requirements(tmpdir.joinpath("req1.txt"), finder=finder,
session=PipSession(), options=options))
assert finder.index_urls == ['url1', 'url2']
def test_multiple_appending_options(self, tmpdir, finder, options):
with open(tmpdir.join("req1.txt"), "w") as fp:
fp.write("--extra-index-url url1 \n")
fp.write("--extra-index-url url2 ")
list(parse_requirements(tmpdir.join("req1.txt"), finder=finder,
session=PipSession(), options=options))
assert finder.index_urls == ['url1', 'url2']
def parse_requirement_file(filename):
from pip._internal.req.constructors import install_req_from_parsed_requirement
finder = get_finder([])
ireqs = [
install_req_from_parsed_requirement(pr)
for pr in parse_requirements(filename, finder.session, finder)
]
return ireqs, finder
def test_skip_regex(self, tmpdir, finder, options):
options.skip_requirements_regex = '.*Bad.*'
with open(tmpdir.joinpath("req1.txt"), "w") as fp:
fp.write("--extra-index-url Bad \n")
fp.write("--extra-index-url Good ")
list(parse_requirements(tmpdir.joinpath("req1.txt"), finder=finder,
options=options, session=PipSession()))
assert finder.index_urls == ['Good']
def find_required_modules(options):
explicit = set()
for requirement in parse_requirements('requirements.txt',
session=PipSession()):
if options.ignore_reqs(requirement):
log.debug('ignoring requirement: %s', requirement.name)
else:
log.debug('found requirement: %s', requirement.name)
explicit.add(canonicalize_name(requirement.name))
return explicit
def test_skip_regex(self, tmpdir, finder, options):
options.skip_requirements_regex = '.*Bad.*'
with open(tmpdir.join("req1.txt"), "w") as fp:
fp.write("--extra-index-url Bad \n")
fp.write("--extra-index-url Good ")
list(parse_requirements(tmpdir.join("req1.txt"), finder=finder,
options=options, session=PipSession()))
assert finder.index_urls == ['Good']
def get_requirements(requirement):
"""Parse a requirement file.
Args:
requirement: path to requirement file
Returns:
list[InstallRequirements]: list of InstallRequirement
"""
session = PipSession()
return parse_requirements(requirement, session=session)
def get_requirements(requirement):
"""Parse a requirement file
:param requirement: path to requirement file
:returns: list of InstallRequirement
:rtype: list[InstallRequirements]
"""
session = PipSession()
return parse_requirements(requirement, session=session)
def test_remote_reqs_parse(self):
"""
Test parsing a simple remote requirements file
"""
# this requirements file just contains a comment previously this has
# failed in py3: https://github.com/pypa/pip/issues/760
for req in parse_requirements(
'https://raw.githubusercontent.com/pypa/'
'pip-test-package/master/'
'tests/req_just_comment.txt', session=PipSession()):
pass
def test_remote_reqs_parse(self):
"""
Test parsing a simple remote requirements file
"""
# this requirements file just contains a comment previously this has
# failed in py3: https://github.com/pypa/pip/issues/760
for req in parse_requirements(
'https://raw.githubusercontent.com/pypa/'
'pip-test-package/master/'
'tests/req_just_comment.txt', session=PipSession()):
pass
def test_req_file_parse_comment_start_of_line(self, tmpdir, finder):
"""
Test parsing comments in a requirements file
"""
with open(tmpdir.joinpath("req1.txt"), "w") as fp:
fp.write("# Comment ")
reqs = list(parse_requirements(tmpdir.joinpath("req1.txt"), finder,
session=PipSession()))
assert not reqs
def test_req_file_parse_comment_start_of_line(self, tmpdir, finder):
"""
Test parsing comments in a requirements file
"""
with open(tmpdir.join("req1.txt"), "w") as fp:
fp.write("# Comment ")
reqs = list(parse_requirements(tmpdir.join("req1.txt"), finder,
session=PipSession()))
assert not reqs
def test_req_file_parse_egginfo_end_of_line_with_url(self, tmpdir, finder):
"""
Test parsing comments in a requirements file
"""
with open(tmpdir.join("req1.txt"), "w") as fp:
fp.write("https://example.com/foo.tar.gz#egg=wat")
reqs = list(parse_requirements(tmpdir.join("req1.txt"), finder,
session=PipSession()))
assert len(reqs) == 1
assert reqs[0].name == "wat"
def test_req_file_parse_egginfo_end_of_line_with_url(self, tmpdir, finder):
"""
Test parsing comments in a requirements file
"""
with open(tmpdir.joinpath("req1.txt"), "w") as fp:
fp.write("https://example.com/foo.tar.gz#egg=wat")
reqs = list(parse_requirements(tmpdir.joinpath("req1.txt"), finder,
session=PipSession()))
assert len(reqs) == 1
assert reqs[0].name == "wat"
def find_required_modules(options):
explicit = set()
for requirement in parse_requirements('requirements.txt',
session=PipSession()):
if not hasattr(requirement, "name"):
from pip._internal.req.constructors import install_req_from_line
requirement = install_req_from_line(requirement.requirement)
if options.ignore_reqs(requirement):
log.debug('ignoring requirement: %s', requirement.name)
else:
log.debug('found requirement: %s', requirement.name)
explicit.add(canonicalize_name(requirement.name))
return explicit
def test_nested_constraints_file(self, monkeypatch, tmpdir):
req_name = 'hello'
req_file = tmpdir / 'parent' / 'req_file.txt'
req_file.parent.mkdir()
req_file.write_text('-c reqs.txt')
req_file.parent.joinpath('reqs.txt').write_text(req_name)
monkeypatch.chdir(str(tmpdir))
reqs = list(
parse_requirements('./parent/req_file.txt', session=session))
assert len(reqs) == 1
assert reqs[0].name == req_name
assert reqs[0].constraint
def test_read_file_url(tmp_path: Path, session: PipSession) -> None:
reqs = tmp_path.joinpath("requirements.txt")
reqs.write_text("foo")
result = list(parse_requirements(reqs.as_posix(), session))
assert len(result) == 1, result
assert result[0].requirement == "foo"
# The comes_from value has three parts: -r or -c flag, path, and line.
# The path value in the middle needs some special logic due to our path
# normalization logic.
assert result[0].comes_from[:3] == "-r "
assert result[0].comes_from[-9:] == " (line 1)"
assert os.path.samefile(result[0].comes_from[3:-9], str(reqs))
def handle(self, *args, **options):
self.style = color_style()
self.options = options
if options["requirements"]:
req_files = options["requirements"]
elif os.path.exists("requirements.txt"):
req_files = ["requirements.txt"]
elif os.path.exists("requirements"):
req_files = [
"requirements/{0}".format(f) for f in os.listdir("requirements")
if os.path.isfile(os.path.join("requirements", f)) and f.lower().endswith(".txt")
]
elif os.path.exists("requirements-dev.txt"):
req_files = ["requirements-dev.txt"]
elif os.path.exists("requirements-prod.txt"):
req_files = ["requirements-prod.txt"]
else:
raise CommandError("Requirements file(s) not found")
self.reqs = {}
with PipSession() as session:
for filename in req_files:
for req in parse_requirements(filename, session=session):
name = req.name if req.name else req.link.filename
# url attribute changed to link in pip version 6.1.0 and above
if LooseVersion(pip.__version__) > LooseVersion('6.0.8'):
self.reqs[name] = {
"pip_req": req,
"url": req.link,
}
else:
self.reqs[name] = {
"pip_req": req,
"url": req.url,
}
if options["github_api_token"]:
self.github_api_token = options["github_api_token"]
elif os.environ.get("GITHUB_API_TOKEN"):
self.github_api_token = os.environ.get("GITHUB_API_TOKEN")
else:
self.github_api_token = None # only 50 requests per hour
self.check_pypi()
if HAS_REQUESTS:
self.check_github()
else:
self.stdout.write(self.style.ERROR("Cannot check github urls. The requests library is not installed. ( pip install requests )"))
self.check_other()
def TestOneInput(input_bytes):
with open("temp.req", "wb") as fd:
fd.write(input_bytes)
try:
[
_ for _ in pipreq.parse_requirements("temp.req",
PipSession(),
finder=None,
options=None,
constraint=None)
]
except UnicodeDecodeError:
# Catch this because I think it's a user issue if Unicode exceptions happen
None
except RequirementsFileParseError:
# Exception thrown by the requirements reader
None
def test_relative_local_nested_req_files(self, session, monkeypatch,
tmpdir):
"""
Test a relative nested req file path is joined with the req file dir
"""
req_name = 'hello'
req_file = tmpdir / 'parent' / 'req_file.txt'
req_file.parent.mkdir()
req_file.write_text('-r reqs.txt')
req_file.parent.joinpath('reqs.txt').write_text(req_name)
monkeypatch.chdir(str(tmpdir))
reqs = list(
parse_requirements('./parent/req_file.txt', session=session))
assert len(reqs) == 1
assert reqs[0].name == req_name
assert not reqs[0].constraint
def parse_reqfile(
filename,
session,
finder=None,
options=None,
constraint=False,
isolated=False,
):
# Wrap parse_requirements/install_req_from_parsed_requirement to
# avoid having to write the same chunk of code in lots of tests.
for parsed_req in parse_requirements(
filename, session, finder=finder,
options=options, constraint=constraint,
):
yield install_req_from_parsed_requirement(
parsed_req,
isolated=isolated
)
def find_missing_reqs(options):
# 1. find files used by imports in the code (as best we can without
# executing)
used_modules = common.find_imported_modules(options)
# 2. find which packages provide which files
installed_files = {}
all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
for package in search_packages_info(all_pkgs):
log.debug('installed package: %s (at %s)', package['name'],
package['location'])
for file in package.get('files', []) or []:
path = os.path.realpath(os.path.join(package['location'], file))
installed_files[path] = package['name']
package_path = common.is_package_file(path)
if package_path:
# we've seen a package file so add the bare package directory
# to the installed list as well as we might want to look up
# a package by its directory path later
installed_files[package_path] = package['name']
# 3. match imported modules against those packages
used = collections.defaultdict(list)
for modname, info in used_modules.items():
# probably standard library if it's not in the files list
if info.filename in installed_files:
used_name = canonicalize_name(installed_files[info.filename])
log.debug('used module: %s (from package %s)', modname,
installed_files[info.filename])
used[used_name].append(info)
else:
log.debug(
'used module: %s (from file %s, assuming stdlib or local)',
modname, info.filename)
# 4. compare with requirements.txt
explicit = set()
for requirement in parse_requirements('requirements.txt',
session=PipSession()):
log.debug('found requirement: %s', requirement.name)
explicit.add(canonicalize_name(requirement.name))
return [(name, used[name]) for name in used
if name not in explicit]
def test_expand_missing_env_variables(self, tmpdir, finder):
req_url = (
'https://${NON_EXISTENT_VARIABLE}:$WRONG_FORMAT@'
'%WINDOWS_FORMAT%github.com/user/repo/archive/master.zip'
)
with open(tmpdir.join('req1.txt'), 'w') as fp:
fp.write(req_url)
with patch('pip._internal.req.req_file.os.getenv') as getenv:
getenv.return_value = ''
reqs = list(parse_requirements(
tmpdir.join('req1.txt'),
finder=finder,
session=PipSession()
))
assert len(reqs) == 1, \
'parsing requirement file with env variable failed'
assert reqs[0].link.url == req_url, \
'ignoring invalid env variable in req file failed'
def populate_requirement_set(requirement_set, # type: RequirementSet
args, # type: List[str]
options, # type: Values
finder, # type: PackageFinder
session, # type: PipSession
name, # type: str
wheel_cache # type: Optional[WheelCache]
):
# type: (...) -> None
"""
Marshal cmd line args into a requirement set.
"""
# NOTE: As a side-effect, options.require_hashes and
# requirement_set.require_hashes may be updated
for filename in options.constraints:
for req_to_add in parse_requirements(
filename,
constraint=True, finder=finder, options=options,
session=session, wheel_cache=wheel_cache):
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
for req in args:
req_to_add = install_req_from_line(
req, None, isolated=options.isolated_mode,
use_pep517=options.use_pep517,
wheel_cache=wheel_cache
)
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
for req in options.editables:
req_to_add = install_req_from_editable(
req,
isolated=options.isolated_mode,
use_pep517=options.use_pep517,
wheel_cache=wheel_cache
)
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
for filename in options.requirements:
for req_to_add in parse_requirements(
filename,
finder=finder, options=options, session=session,
wheel_cache=wheel_cache,
use_pep517=options.use_pep517):
req_to_add.is_direct = True
requirement_set.add_requirement(req_to_add)
# If --require-hashes was a line in a requirements file, tell
# RequirementSet about it:
requirement_set.require_hashes = options.require_hashes
if not (args or options.editables or options.requirements):
opts = {'name': name}
if options.find_links:
raise CommandError(
'You must give at least one requirement to %(name)s '
'(maybe you meant "pip %(name)s %(links)s"?)' %
dict(opts, links=' '.join(options.find_links)))
else:
raise CommandError(
'You must give at least one requirement to %(name)s '
'(see "pip help %(name)s")' % opts)
def populate_requirement_set(requirement_set, args, options, finder,
session, name, wheel_cache):
"""
Marshal cmd line args into a requirement set.
"""
# NOTE: As a side-effect, options.require_hashes and
# requirement_set.require_hashes may be updated
for filename in options.constraints:
for req in parse_requirements(
filename,
constraint=True, finder=finder, options=options,
session=session, wheel_cache=wheel_cache):
requirement_set.add_requirement(req)
for req in args:
requirement_set.add_requirement(
InstallRequirement.from_line(
req, None, isolated=options.isolated_mode,
wheel_cache=wheel_cache
)
)
for req in options.editables:
requirement_set.add_requirement(
InstallRequirement.from_editable(
req,
isolated=options.isolated_mode,
wheel_cache=wheel_cache
)
)
for filename in options.requirements:
for req in parse_requirements(
filename,
finder=finder, options=options, session=session,
wheel_cache=wheel_cache):
requirement_set.add_requirement(req)
# If --require-hashes was a line in a requirements file, tell
# RequirementSet about it:
requirement_set.require_hashes = options.require_hashes
if not (args or options.editables or options.requirements):
opts = {'name': name}
if options.find_links:
raise CommandError(
'You must give at least one requirement to %(name)s '
'(maybe you meant "pip %(name)s %(links)s"?)' %
dict(opts, links=' '.join(options.find_links)))
else:
raise CommandError(
'You must give at least one requirement to %(name)s '
'(see "pip help %(name)s")' % opts)
# On Windows, any operation modifying pip should be run as:
# python -m pip ...
# See https://github.com/pypa/pip/issues/1299 for more discussion
should_show_use_python_msg = (
WINDOWS and
requirement_set.has_requirement('pip') and
"pip" in os.path.basename(sys.argv[0])
)
if should_show_use_python_msg:
new_command = [
sys.executable, "-m", "pip"
] + sys.argv[1:]
raise CommandError(
'To modify pip, please run the following command:\n{}'
.format(" ".join(new_command))
)
from setuptools import setup, find_packages
from pip._internal.req.req_file import parse_requirements
from pip._internal.download import PipSession
from os import path
# Lists of requirements and dependency links which are needed during runtime, testing and setup
install_requires = []
tests_require = []
dependency_links = []
# Inject requirements from requirements.txt into setup.py
requirements_file = parse_requirements(path.join('requirements', 'requirements.txt'), session=PipSession())
for req in requirements_file:
install_requires.append(str(req.req))
if req.link:
dependency_links.append(str(req.link))
# Inject test requirements from requirements_test.txt into setup.py
requirements_test_file = parse_requirements(path.join('requirements', 'requirements_test.txt'), session=PipSession())
for req in requirements_test_file:
tests_require.append(str(req.req))
if req.link:
dependency_links.append(str(req.link))
# Became django-statsd-unleashed because django-statsd and django-statsd-mozilla are taken on Pypi. ;)
setup(
name='django-statsd-unleashed',
version='1.0.5',
