def populate_requirement_set(requirement_set, args, options, finder, session, name, wheel_cache): """ Marshal cmd line args into a requirement set. """ # NOTE: As a side-effect, options.require_hashes and # requirement_set.require_hashes may be updated for filename in options.constraints: for req_to_add in parse_requirements( filename, constraint=True, finder=finder, options=options, session=session, wheel_cache=wheel_cache, ): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for req in args: req_to_add = install_req_from_line(req, None, isolated=options.isolated_mode, wheel_cache=wheel_cache) req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for req in options.editables: req_to_add = install_req_from_editable( req, isolated=options.isolated_mode, wheel_cache=wheel_cache) req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for filename in options.requirements: for req_to_add in parse_requirements( filename, finder=finder, options=options, session=session, wheel_cache=wheel_cache, ): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) # If --require-hashes was a line in a requirements file, tell # RequirementSet about it: requirement_set.require_hashes = options.require_hashes if not (args or options.editables or options.requirements): opts = {"name": name} if options.find_links: raise CommandError( "You must give at least one requirement to %(name)s " '(maybe you meant "pip %(name)s %(links)s"?)' % dict(opts, links=" ".join(options.find_links))) else: raise CommandError( "You must give at least one requirement to %(name)s " '(see "pip help %(name)s")' % opts)
def python_validator(data): """Very simple Python requirements.txt validator.""" requirements_txt = StringIO() requirements_txt.write(data) try: parse_requirements(requirements_txt, session="requirements.txt") except RequirementsFileParseError: return False return True
def populate_requirement_set(requirement_set, args, options, finder, session, name, wheel_cache): """ Marshal cmd line args into a requirement set. """ # NOTE: As a side-effect, options.require_hashes and # requirement_set.require_hashes may be updated for filename in options.constraints: for req_to_add in parse_requirements( filename, constraint=True, finder=finder, options=options, session=session, wheel_cache=wheel_cache): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for req in args: req_to_add = InstallRequirement.from_line( req, None, isolated=options.isolated_mode, wheel_cache=wheel_cache ) req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for req in options.editables: req_to_add = InstallRequirement.from_editable( req, isolated=options.isolated_mode, wheel_cache=wheel_cache ) req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for filename in options.requirements: for req_to_add in parse_requirements( filename, finder=finder, options=options, session=session, wheel_cache=wheel_cache): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) # If --require-hashes was a line in a requirements file, tell # RequirementSet about it: requirement_set.require_hashes = options.require_hashes if not (args or options.editables or options.requirements): opts = {'name': name} if options.find_links: raise CommandError( 'You must give at least one requirement to %(name)s ' '(maybe you meant "pip %(name)s %(links)s"?)' % dict(opts, links=' '.join(options.find_links))) else: raise CommandError( 'You must give at least one requirement to %(name)s ' '(see "pip help %(name)s")' % opts)
def test_req_file_no_finder(self, tmpdir): """ Test parsing a requirements file without a finder """ with open(tmpdir.joinpath("req.txt"), "w") as fp: fp.write(""" --find-links https://example.com/ --index-url https://example.com/ --extra-index-url https://two.example.com/ --no-use-wheel --no-index """) parse_requirements(tmpdir.joinpath("req.txt"), session=PipSession())
def test_req_file_no_finder(self, tmpdir): """ Test parsing a requirements file without a finder """ with open(tmpdir.join("req.txt"), "w") as fp: fp.write(""" --find-links https://example.com/ --index-url https://example.com/ --extra-index-url https://two.example.com/ --no-use-wheel --no-index """) parse_requirements(tmpdir.join("req.txt"), session=PipSession())
def test_install_requirements_with_options(self, tmpdir, finder, session, options): global_option = '--dry-run' install_option = '--prefix=/opt' content = ''' --only-binary :all: INITools==2.0 --global-option="{global_option}" \ --install-option "{install_option}" '''.format(global_option=global_option, install_option=install_option) with requirements_file(content, tmpdir) as reqs_file: req = next( parse_requirements(reqs_file.abspath, finder=finder, options=options, session=session)) req.source_dir = os.curdir with patch.object(subprocess, 'Popen') as popen: popen.return_value.stdout.readline.return_value = b"" try: req.install([]) except Exception: pass last_call = popen.call_args_list[-1] args = last_call[0][0] assert (0 < args.index(global_option) < args.index('install') < args.index(install_option)) assert options.format_control.no_binary == {':all:'} assert options.format_control.only_binary == set()
def merge_requirements(files): requirements = defaultdict(lambda: Requirement()) links = set() for filename in files: for requirement in parse_requirements(filename, session=PipSession()): if not hasattr(requirement.req, 'name'): links.add(requirement.link.url) break name = requirement.req.name specifiers = requirement.req.specifier extras = requirement.req.extras requirements[name].extras |= set(extras) requirements[name].specifiers |= set(specifiers) result = [] for key, value in requirements.items(): if not value.extras: result.append("%s %s" % (key, ",".join(map(str, value.specifiers)))) else: result.append("%s [%s] %s" % (key, ",".join(map( str, value.extras)), ",".join(map(str, value.specifiers)))) for link in links: result.append(link) return "\n".join(result)
def test_req_file_parse_no_only_binary(self, data, finder): list( parse_requirements(data.reqfiles.join("supported_options2.txt"), finder, session=PipSession())) expected = FormatControl({'fred'}, {'wilma'}) assert finder.format_control == expected
def test_install_requirements_with_options(self, tmpdir, finder, session, options): global_option = '--dry-run' install_option = '--prefix=/opt' content = ''' --only-binary :all: INITools==2.0 --global-option="{global_option}" \ --install-option "{install_option}" '''.format(global_option=global_option, install_option=install_option) with requirements_file(content, tmpdir) as reqs_file: req = next(parse_requirements(reqs_file.abspath, finder=finder, options=options, session=session)) req.source_dir = os.curdir with patch.object(subprocess, 'Popen') as popen: popen.return_value.stdout.readline.return_value = b"" try: req.install([]) except: pass last_call = popen.call_args_list[-1] args = last_call[0][0] assert ( 0 < args.index(global_option) < args.index('install') < args.index(install_option) ) assert options.format_control.no_binary == {':all:'} assert options.format_control.only_binary == set()
def test_absolute_http_nested_req_file_in_local( self, session, monkeypatch, tmpdir ): """ Test a nested req file url in a local req file """ req_name = 'hello' req_file = tmpdir / 'req_file.txt' nested_req_file = 'http://me.com/me/req_file.txt' def get_file_content(filename, *args, **kwargs): if filename == str(req_file): return None, '-r {}'.format(nested_req_file) elif filename == nested_req_file: return None, req_name assert False, 'Unexpected file requested {}'.format(filename) monkeypatch.setattr( pip._internal.req.req_file, 'get_file_content', get_file_content ) result = list(parse_requirements(req_file, session=session)) assert len(result) == 1 assert result[0].name == req_name assert not result[0].constraint
def test_expand_existing_env_variables(self, tmpdir, finder): template = ( 'https://%s:[email protected]/user/%s/archive/master.zip') env_vars = ( ('GITHUB_TOKEN', 'notarealtoken'), ('DO_12_FACTOR', 'awwyeah'), ) with open(tmpdir.joinpath('req1.txt'), 'w') as fp: fp.write(template % tuple(['${%s}' % k for k, _ in env_vars])) with patch('pip._internal.req.req_file.os.getenv') as getenv: getenv.side_effect = lambda n: dict(env_vars)[n] reqs = list( parse_requirements(tmpdir.joinpath('req1.txt'), finder=finder, session=PipSession())) assert len(reqs) == 1, \ 'parsing requirement file with env variable failed' expected_url = template % tuple([v for _, v in env_vars]) assert reqs[0].link.url == expected_url, \ 'variable expansion in req file failed'
def test_expand_existing_env_variables(self, tmpdir, finder): template = ( 'https://%s:[email protected]/user/%s/archive/master.zip' ) env_vars = ( ('GITHUB_TOKEN', 'notarealtoken'), ('DO_12_FACTOR', 'awwyeah'), ) with open(tmpdir.join('req1.txt'), 'w') as fp: fp.write(template % tuple(['${%s}' % k for k, _ in env_vars])) with patch('pip._internal.req.req_file.os.getenv') as getenv: getenv.side_effect = lambda n: dict(env_vars)[n] reqs = list(parse_requirements( tmpdir.join('req1.txt'), finder=finder, session=PipSession() )) assert len(reqs) == 1, \ 'parsing requirement file with env variable failed' expected_url = template % tuple([v for _, v in env_vars]) assert reqs[0].link.url == expected_url, \ 'variable expansion in req file failed'
def process_line( line, filename, line_number, finder=None, options=None, session=None, constraint=False, ): if session is None: session = PipSession() prefix = '\n' * (line_number - 1) path = tmpdir.joinpath(filename) path.parent.mkdir(exist_ok=True) path.write_text(prefix + line) monkeypatch.chdir(str(tmpdir)) return list( parse_requirements( filename, finder=finder, options=options, session=session, constraint=constraint, ))
def test_req_file_parse_no_only_binary(self, data, finder): list(parse_requirements( data.reqfiles.join("supported_options2.txt"), finder, session=PipSession())) expected = pip._internal.index.FormatControl( {'fred'}, {'wilma'}) assert finder.format_control == expected
def find_required_modules(options, requirements_filename: str): explicit = set() for requirement in parse_requirements(requirements_filename, session=PipSession()): try: requirement_name = requirement.name # The type of "requirement" changed between pip versions. # We exclude the "except" from coverage so that on any pip version we # can report 100% coverage. except AttributeError: # pragma: no cover from pip._internal.req.constructors import install_req_from_line requirement_name = install_req_from_line( requirement.requirement, ).name if options.ignore_reqs(requirement): log.debug('ignoring requirement: %s', requirement_name) continue if options.skip_incompatible: requirement_string = requirement.requirement if not has_compatible_markers(requirement_string): log.debug('ignoring requirement (incompatible environment ' 'marker): %s', requirement_string) continue log.debug('found requirement: %s', requirement_name) explicit.add(canonicalize_name(requirement_name)) return explicit
def test_join_lines(self, tmpdir, finder): with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write("--extra-index-url url1 \\\n--extra-index-url url2") list(parse_requirements(tmpdir.joinpath("req1.txt"), finder=finder, session=PipSession())) assert finder.index_urls == ['url1', 'url2']
def test_join_lines(self, tmpdir, finder): with open(tmpdir.join("req1.txt"), "w") as fp: fp.write("--extra-index-url url1 \\\n--extra-index-url url2") list(parse_requirements(tmpdir.join("req1.txt"), finder=finder, session=PipSession())) assert finder.index_urls == ['url1', 'url2']
def test_req_file_parse_no_only_binary(self, data, finder): list( parse_requirements(data.reqfiles.join("supported_options2.txt"), finder, session=PipSession())) expected = pip._internal.index.FormatControl(set(['fred']), set(['wilma'])) assert finder.format_control == expected
def handle(self, *args, **options): self.style = color_style() self.options = options if options["requirements"]: req_files = options["requirements"] elif os.path.exists("requirements.txt"): req_files = ["requirements.txt"] elif os.path.exists("requirements"): req_files = [ "requirements/{0}".format(f) for f in os.listdir("requirements") if os.path.isfile(os.path.join("requirements", f)) and f.lower().endswith(".txt") ] elif os.path.exists("requirements-dev.txt"): req_files = ["requirements-dev.txt"] elif os.path.exists("requirements-prod.txt"): req_files = ["requirements-prod.txt"] else: raise CommandError("Requirements file(s) not found") self.reqs = {} with PipSession() as session: for filename in req_files: for req in parse_requirements(filename, session=session): if not isinstance(req, InstallRequirement): req = install_req_from_line(req.requirement) name = req.name if req.name else req.link.filename # url attribute changed to link in pip version 6.1.0 and above if LooseVersion(pip.__version__) > LooseVersion('6.0.8'): self.reqs[name] = { "pip_req": req, "url": req.link, } else: self.reqs[name] = { "pip_req": req, "url": req.url, } if options["github_api_token"]: self.github_api_token = options["github_api_token"] elif os.environ.get("GITHUB_API_TOKEN"): self.github_api_token = os.environ.get("GITHUB_API_TOKEN") else: self.github_api_token = None # only 50 requests per hour self.check_pypi() if HAS_REQUESTS: self.check_github() else: self.stdout.write( self.style.ERROR( "Cannot check github urls. The requests library is not installed. ( pip install requests )" )) self.check_other()
def _parse_python(spec): """Parse PyPI specification of a single dependency. :param spec: str, for example "Django>=1.5,<1.8" :return: [Django [[('>=', '1.5'), ('<', '1.8')]]] """ def _extract_op_version(spec): # https://www.python.org/dev/peps/pep-0440/#compatible-release if spec.operator == '~=': version = _version_split(spec.version) if len(version) > 1: # ignore pre-release, post-release or developmental release while not version[-1].isdigit(): del version[-1] del version[-1] # will increase the last but one in next line version[-1] = str(int(version[-1]) + 1) else: raise ValueError('%r must not be used with %r' % (spec.operator, spec.version)) return [('>=', spec.version), ('<', '.'.join(version))] # Trailing .* is permitted per # https://www.python.org/dev/peps/pep-0440/#version-matching elif spec.operator == '==' and spec.version.endswith('.*'): try: result = check_output(['/usr/bin/semver-ranger', spec.version], universal_newlines=True).strip() gte, lt = result.split() return [('>=', gte.lstrip('>=')), ('<', lt.lstrip('<'))] except ValueError: logger.info("couldn't resolve ==%s", spec.version) return spec.operator, spec.version # https://www.python.org/dev/peps/pep-0440/#arbitrary-equality # Use of this operator is heavily discouraged, so just convert it to 'Version matching' elif spec.operator == '===': return '==', spec.version else: return spec.operator, spec.version def _get_pip_spec(requirements): """There's no `specs` field In Pip 8+, take info from `specifier` field.""" if hasattr(requirements, 'specs'): return requirements.specs elif hasattr(requirements, 'specifier'): specs = [_extract_op_version(spec) for spec in requirements.specifier] if len(specs) == 0: specs = [('>=', '0.0.0')] elif len(specs) > 1: specs = [specs] return specs # create a temporary file and store the spec there since # `parse_requirements` requires a file with NamedTemporaryFile(mode='w+', suffix='pysolve') as f: f.write(spec) f.flush() parsed = parse_requirements(f.name, session=f.name) dependency = [Dependency(x.name, _get_pip_spec(x.req)) for x in parsed].pop() return dependency
def find_missing_reqs(options, requirements_filename): # 1. find files used by imports in the code (as best we can without # executing) used_modules = common.find_imported_modules(options) # 2. find which packages provide which files installed_files = {} all_pkgs = (pkg.project_name for pkg in get_installed_distributions()) for package in search_packages_info(all_pkgs): log.debug('installed package: %s (at %s)', package['name'], package['location']) for package_file in package.get('files', []) or []: path = os.path.realpath( os.path.join(package['location'], package_file), ) installed_files[path] = package['name'] package_path = common.is_package_file(path) if package_path: # we've seen a package file so add the bare package directory # to the installed list as well as we might want to look up # a package by its directory path later installed_files[package_path] = package['name'] # 3. match imported modules against those packages used = collections.defaultdict(list) for modname, info in used_modules.items(): # probably standard library if it's not in the files list if info.filename in installed_files: used_name = canonicalize_name(installed_files[info.filename]) log.debug('used module: %s (from package %s)', modname, installed_files[info.filename]) used[used_name].append(info) else: log.debug( 'used module: %s (from file %s, assuming stdlib or local)', modname, info.filename) # 4. compare with requirements.txt explicit = set() for requirement in parse_requirements( requirements_filename, session=PipSession(), ): try: requirement_name = requirement.name # The type of "requirement" changed between pip versions. # We exclude the "except" from coverage so that on any pip version we # can report 100% coverage. except AttributeError: # pragma: no cover from pip._internal.req.constructors import install_req_from_line requirement_name = install_req_from_line( requirement.requirement, ).name log.debug('found requirement: %s', requirement_name) explicit.add(canonicalize_name(requirement_name)) return [(name, used[name]) for name in used if name not in explicit]
def test_multiple_appending_options(self, tmpdir, finder, options): with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write("--extra-index-url url1 \n") fp.write("--extra-index-url url2 ") list(parse_requirements(tmpdir.joinpath("req1.txt"), finder=finder, session=PipSession(), options=options)) assert finder.index_urls == ['url1', 'url2']
def test_multiple_appending_options(self, tmpdir, finder, options): with open(tmpdir.join("req1.txt"), "w") as fp: fp.write("--extra-index-url url1 \n") fp.write("--extra-index-url url2 ") list(parse_requirements(tmpdir.join("req1.txt"), finder=finder, session=PipSession(), options=options)) assert finder.index_urls == ['url1', 'url2']
def parse_requirement_file(filename): from pip._internal.req.constructors import install_req_from_parsed_requirement finder = get_finder([]) ireqs = [ install_req_from_parsed_requirement(pr) for pr in parse_requirements(filename, finder.session, finder) ] return ireqs, finder
def test_skip_regex(self, tmpdir, finder, options): options.skip_requirements_regex = '.*Bad.*' with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write("--extra-index-url Bad \n") fp.write("--extra-index-url Good ") list(parse_requirements(tmpdir.joinpath("req1.txt"), finder=finder, options=options, session=PipSession())) assert finder.index_urls == ['Good']
def find_required_modules(options): explicit = set() for requirement in parse_requirements('requirements.txt', session=PipSession()): if options.ignore_reqs(requirement): log.debug('ignoring requirement: %s', requirement.name) else: log.debug('found requirement: %s', requirement.name) explicit.add(canonicalize_name(requirement.name)) return explicit
def test_skip_regex(self, tmpdir, finder, options): options.skip_requirements_regex = '.*Bad.*' with open(tmpdir.join("req1.txt"), "w") as fp: fp.write("--extra-index-url Bad \n") fp.write("--extra-index-url Good ") list(parse_requirements(tmpdir.join("req1.txt"), finder=finder, options=options, session=PipSession())) assert finder.index_urls == ['Good']
def get_requirements(requirement): """Parse a requirement file. Args: requirement: path to requirement file Returns: list[InstallRequirements]: list of InstallRequirement """ session = PipSession() return parse_requirements(requirement, session=session)
def get_requirements(requirement): """Parse a requirement file :param requirement: path to requirement file :returns: list of InstallRequirement :rtype: list[InstallRequirements] """ session = PipSession() return parse_requirements(requirement, session=session)
def test_remote_reqs_parse(self): """ Test parsing a simple remote requirements file """ # this requirements file just contains a comment previously this has # failed in py3: https://github.com/pypa/pip/issues/760 for req in parse_requirements( 'https://raw.githubusercontent.com/pypa/' 'pip-test-package/master/' 'tests/req_just_comment.txt', session=PipSession()): pass
def test_req_file_parse_comment_start_of_line(self, tmpdir, finder): """ Test parsing comments in a requirements file """ with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write("# Comment ") reqs = list(parse_requirements(tmpdir.joinpath("req1.txt"), finder, session=PipSession())) assert not reqs
def test_req_file_parse_comment_start_of_line(self, tmpdir, finder): """ Test parsing comments in a requirements file """ with open(tmpdir.join("req1.txt"), "w") as fp: fp.write("# Comment ") reqs = list(parse_requirements(tmpdir.join("req1.txt"), finder, session=PipSession())) assert not reqs
def test_req_file_parse_egginfo_end_of_line_with_url(self, tmpdir, finder): """ Test parsing comments in a requirements file """ with open(tmpdir.join("req1.txt"), "w") as fp: fp.write("https://example.com/foo.tar.gz#egg=wat") reqs = list(parse_requirements(tmpdir.join("req1.txt"), finder, session=PipSession())) assert len(reqs) == 1 assert reqs[0].name == "wat"
def test_req_file_parse_egginfo_end_of_line_with_url(self, tmpdir, finder): """ Test parsing comments in a requirements file """ with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write("https://example.com/foo.tar.gz#egg=wat") reqs = list(parse_requirements(tmpdir.joinpath("req1.txt"), finder, session=PipSession())) assert len(reqs) == 1 assert reqs[0].name == "wat"
def find_required_modules(options): explicit = set() for requirement in parse_requirements('requirements.txt', session=PipSession()): if not hasattr(requirement, "name"): from pip._internal.req.constructors import install_req_from_line requirement = install_req_from_line(requirement.requirement) if options.ignore_reqs(requirement): log.debug('ignoring requirement: %s', requirement.name) else: log.debug('found requirement: %s', requirement.name) explicit.add(canonicalize_name(requirement.name)) return explicit
def test_nested_constraints_file(self, monkeypatch, tmpdir): req_name = 'hello' req_file = tmpdir / 'parent' / 'req_file.txt' req_file.parent.mkdir() req_file.write_text('-c reqs.txt') req_file.parent.joinpath('reqs.txt').write_text(req_name) monkeypatch.chdir(str(tmpdir)) reqs = list( parse_requirements('./parent/req_file.txt', session=session)) assert len(reqs) == 1 assert reqs[0].name == req_name assert reqs[0].constraint
def test_read_file_url(tmp_path: Path, session: PipSession) -> None: reqs = tmp_path.joinpath("requirements.txt") reqs.write_text("foo") result = list(parse_requirements(reqs.as_posix(), session)) assert len(result) == 1, result assert result[0].requirement == "foo" # The comes_from value has three parts: -r or -c flag, path, and line. # The path value in the middle needs some special logic due to our path # normalization logic. assert result[0].comes_from[:3] == "-r " assert result[0].comes_from[-9:] == " (line 1)" assert os.path.samefile(result[0].comes_from[3:-9], str(reqs))
def handle(self, *args, **options): self.style = color_style() self.options = options if options["requirements"]: req_files = options["requirements"] elif os.path.exists("requirements.txt"): req_files = ["requirements.txt"] elif os.path.exists("requirements"): req_files = [ "requirements/{0}".format(f) for f in os.listdir("requirements") if os.path.isfile(os.path.join("requirements", f)) and f.lower().endswith(".txt") ] elif os.path.exists("requirements-dev.txt"): req_files = ["requirements-dev.txt"] elif os.path.exists("requirements-prod.txt"): req_files = ["requirements-prod.txt"] else: raise CommandError("Requirements file(s) not found") self.reqs = {} with PipSession() as session: for filename in req_files: for req in parse_requirements(filename, session=session): name = req.name if req.name else req.link.filename # url attribute changed to link in pip version 6.1.0 and above if LooseVersion(pip.__version__) > LooseVersion('6.0.8'): self.reqs[name] = { "pip_req": req, "url": req.link, } else: self.reqs[name] = { "pip_req": req, "url": req.url, } if options["github_api_token"]: self.github_api_token = options["github_api_token"] elif os.environ.get("GITHUB_API_TOKEN"): self.github_api_token = os.environ.get("GITHUB_API_TOKEN") else: self.github_api_token = None # only 50 requests per hour self.check_pypi() if HAS_REQUESTS: self.check_github() else: self.stdout.write(self.style.ERROR("Cannot check github urls. The requests library is not installed. ( pip install requests )")) self.check_other()
def TestOneInput(input_bytes): with open("temp.req", "wb") as fd: fd.write(input_bytes) try: [ _ for _ in pipreq.parse_requirements("temp.req", PipSession(), finder=None, options=None, constraint=None) ] except UnicodeDecodeError: # Catch this because I think it's a user issue if Unicode exceptions happen None except RequirementsFileParseError: # Exception thrown by the requirements reader None
def test_relative_local_nested_req_files(self, session, monkeypatch, tmpdir): """ Test a relative nested req file path is joined with the req file dir """ req_name = 'hello' req_file = tmpdir / 'parent' / 'req_file.txt' req_file.parent.mkdir() req_file.write_text('-r reqs.txt') req_file.parent.joinpath('reqs.txt').write_text(req_name) monkeypatch.chdir(str(tmpdir)) reqs = list( parse_requirements('./parent/req_file.txt', session=session)) assert len(reqs) == 1 assert reqs[0].name == req_name assert not reqs[0].constraint
def parse_reqfile( filename, session, finder=None, options=None, constraint=False, isolated=False, ): # Wrap parse_requirements/install_req_from_parsed_requirement to # avoid having to write the same chunk of code in lots of tests. for parsed_req in parse_requirements( filename, session, finder=finder, options=options, constraint=constraint, ): yield install_req_from_parsed_requirement( parsed_req, isolated=isolated )
def find_missing_reqs(options): # 1. find files used by imports in the code (as best we can without # executing) used_modules = common.find_imported_modules(options) # 2. find which packages provide which files installed_files = {} all_pkgs = (pkg.project_name for pkg in get_installed_distributions()) for package in search_packages_info(all_pkgs): log.debug('installed package: %s (at %s)', package['name'], package['location']) for file in package.get('files', []) or []: path = os.path.realpath(os.path.join(package['location'], file)) installed_files[path] = package['name'] package_path = common.is_package_file(path) if package_path: # we've seen a package file so add the bare package directory # to the installed list as well as we might want to look up # a package by its directory path later installed_files[package_path] = package['name'] # 3. match imported modules against those packages used = collections.defaultdict(list) for modname, info in used_modules.items(): # probably standard library if it's not in the files list if info.filename in installed_files: used_name = canonicalize_name(installed_files[info.filename]) log.debug('used module: %s (from package %s)', modname, installed_files[info.filename]) used[used_name].append(info) else: log.debug( 'used module: %s (from file %s, assuming stdlib or local)', modname, info.filename) # 4. compare with requirements.txt explicit = set() for requirement in parse_requirements('requirements.txt', session=PipSession()): log.debug('found requirement: %s', requirement.name) explicit.add(canonicalize_name(requirement.name)) return [(name, used[name]) for name in used if name not in explicit]
def test_expand_missing_env_variables(self, tmpdir, finder): req_url = ( 'https://${NON_EXISTENT_VARIABLE}:$WRONG_FORMAT@' '%WINDOWS_FORMAT%github.com/user/repo/archive/master.zip' ) with open(tmpdir.join('req1.txt'), 'w') as fp: fp.write(req_url) with patch('pip._internal.req.req_file.os.getenv') as getenv: getenv.return_value = '' reqs = list(parse_requirements( tmpdir.join('req1.txt'), finder=finder, session=PipSession() )) assert len(reqs) == 1, \ 'parsing requirement file with env variable failed' assert reqs[0].link.url == req_url, \ 'ignoring invalid env variable in req file failed'
def populate_requirement_set(requirement_set, # type: RequirementSet args, # type: List[str] options, # type: Values finder, # type: PackageFinder session, # type: PipSession name, # type: str wheel_cache # type: Optional[WheelCache] ): # type: (...) -> None """ Marshal cmd line args into a requirement set. """ # NOTE: As a side-effect, options.require_hashes and # requirement_set.require_hashes may be updated for filename in options.constraints: for req_to_add in parse_requirements( filename, constraint=True, finder=finder, options=options, session=session, wheel_cache=wheel_cache): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for req in args: req_to_add = install_req_from_line( req, None, isolated=options.isolated_mode, use_pep517=options.use_pep517, wheel_cache=wheel_cache ) req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for req in options.editables: req_to_add = install_req_from_editable( req, isolated=options.isolated_mode, use_pep517=options.use_pep517, wheel_cache=wheel_cache ) req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for filename in options.requirements: for req_to_add in parse_requirements( filename, finder=finder, options=options, session=session, wheel_cache=wheel_cache, use_pep517=options.use_pep517): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) # If --require-hashes was a line in a requirements file, tell # RequirementSet about it: requirement_set.require_hashes = options.require_hashes if not (args or options.editables or options.requirements): opts = {'name': name} if options.find_links: raise CommandError( 'You must give at least one requirement to %(name)s ' '(maybe you meant "pip %(name)s %(links)s"?)' % dict(opts, links=' '.join(options.find_links))) else: raise CommandError( 'You must give at least one requirement to %(name)s ' '(see "pip help %(name)s")' % opts)
def populate_requirement_set(requirement_set, args, options, finder, session, name, wheel_cache): """ Marshal cmd line args into a requirement set. """ # NOTE: As a side-effect, options.require_hashes and # requirement_set.require_hashes may be updated for filename in options.constraints: for req in parse_requirements( filename, constraint=True, finder=finder, options=options, session=session, wheel_cache=wheel_cache): requirement_set.add_requirement(req) for req in args: requirement_set.add_requirement( InstallRequirement.from_line( req, None, isolated=options.isolated_mode, wheel_cache=wheel_cache ) ) for req in options.editables: requirement_set.add_requirement( InstallRequirement.from_editable( req, isolated=options.isolated_mode, wheel_cache=wheel_cache ) ) for filename in options.requirements: for req in parse_requirements( filename, finder=finder, options=options, session=session, wheel_cache=wheel_cache): requirement_set.add_requirement(req) # If --require-hashes was a line in a requirements file, tell # RequirementSet about it: requirement_set.require_hashes = options.require_hashes if not (args or options.editables or options.requirements): opts = {'name': name} if options.find_links: raise CommandError( 'You must give at least one requirement to %(name)s ' '(maybe you meant "pip %(name)s %(links)s"?)' % dict(opts, links=' '.join(options.find_links))) else: raise CommandError( 'You must give at least one requirement to %(name)s ' '(see "pip help %(name)s")' % opts) # On Windows, any operation modifying pip should be run as: # python -m pip ... # See https://github.com/pypa/pip/issues/1299 for more discussion should_show_use_python_msg = ( WINDOWS and requirement_set.has_requirement('pip') and "pip" in os.path.basename(sys.argv[0]) ) if should_show_use_python_msg: new_command = [ sys.executable, "-m", "pip" ] + sys.argv[1:] raise CommandError( 'To modify pip, please run the following command:\n{}' .format(" ".join(new_command)) )
from setuptools import setup, find_packages from pip._internal.req.req_file import parse_requirements from pip._internal.download import PipSession from os import path # Lists of requirements and dependency links which are needed during runtime, testing and setup install_requires = [] tests_require = [] dependency_links = [] # Inject requirements from requirements.txt into setup.py requirements_file = parse_requirements(path.join('requirements', 'requirements.txt'), session=PipSession()) for req in requirements_file: install_requires.append(str(req.req)) if req.link: dependency_links.append(str(req.link)) # Inject test requirements from requirements_test.txt into setup.py requirements_test_file = parse_requirements(path.join('requirements', 'requirements_test.txt'), session=PipSession()) for req in requirements_test_file: tests_require.append(str(req.req)) if req.link: dependency_links.append(str(req.link)) # Became django-statsd-unleashed because django-statsd and django-statsd-mozilla are taken on Pypi. ;) setup( name='django-statsd-unleashed', version='1.0.5',