def test_dict_metadata_works():
name = "simple"
version = "0.1.0"
require_a = "a==1.0"
require_b = "b==1.1; extra == 'also_b'"
requires = [require_a, require_b, "c==1.2; extra == 'also_c'"]
extras = ["also_b", "also_c"]
requires_python = ">=3"
metadata = Message()
metadata["Name"] = name
metadata["Version"] = version
for require in requires:
metadata["Requires-Dist"] = require
for extra in extras:
metadata["Provides-Extra"] = extra
metadata["Requires-Python"] = requires_python
inner_metadata = DictMetadata({
"METADATA": ensure_binary(metadata.as_string())
})
dist = DistInfoDistribution(
location="<in-memory>", metadata=inner_metadata, project_name=name
)
assert name == dist.project_name
assert version == dist.version
assert set(extras) == set(dist.extras)
assert [Requirement.parse(require_a)] == dist.requires([])
assert [
Requirement.parse(require_a), Requirement.parse(require_b)
] == dist.requires(["also_b"])
assert metadata.as_string() == get_metadata(dist).as_string()
assert requires_python == get_requires_python(dist)
def is_valid_requirement(cls, requirement):
try:
Requirement.parse(requirement)
except ValueError:
return False
else:
return True
def merge(requirements):
"""
Merges autodetected and explicit requirements together. Autodetected
requirements are pulled out the user's session (i.e. globals()).
Explicit requirements are provided directly by the user. This
function reconciles them and merges them into one set of requirements.
Warnings are given to the user in case of version mismatch.
Because we want to move away from implicitly getting requirements, we warn
the user if there are implicitly detecdet but not explicitly stated
requirements.
"""
implicit_dict = {}
for r in requirements['autodetected']:
implicit_dict[r.project_name] = r
explicit_dict = {}
for r in requirements['modelSpecified']:
if type(r) != str:
explicit_dict[r.project_name] = r
for project_name, exp_req in list(explicit_dict.items()):
# To be polite, we keep the explicit dependencies and add the implicit
# ones to them. We respect versions on the former, except in the case
# of yhat, which should be the installed version.
if project_name in implicit_dict:
imp_req = implicit_dict[project_name]
if exp_req == imp_req:
# we only need one of these, remove the implicit, but don't need
# to warn the user
requirements['autodetected'].remove(Requirement.parse(str(imp_req)))
elif project_name == "yhat":
warn(
"The yhat package can be removed from REQUIREMENTS. "
"It is required and added for you."
)
try:
requirements['autodetected'].remove(Requirement.parse(str(imp_req)))
requirements['modelSpecified'].remove(Requirement.parse(str(exp_req)))
except:
pass
else:
warn(
"You have explicitly %s as a requirement, which differs from %s, "
"which is was implicitly found to be installed locally\n"
"Deploying with explicitly defined package: %s " % (exp_req, imp_req, exp_req)
)
requirements['autodetected'].remove(Requirement.parse(str(imp_req)))
# Loop through the implicit dict and notify users if they haven't explicitly
# specified a requirement
for project_name, imp_req in list(implicit_dict.items()):
if project_name not in explicit_dict:
warn(
"Dependency %s was found with autodetection, but we recommend "
"explicitly stating your requirements." % (imp_req)
)
return requirements
def testRequirmentsString(self):
"""
testRequirmentsString will test a normal requirments list
"""
REQUIREMENTS = ['scikit-learn==0.16.1', 'pandas==0.17.1']
userReqs = parseUserRequirementsList(REQUIREMENTS)
assertReqs = [Requirement.parse('scikit-learn==0.16.1'),
Requirement.parse('pandas==0.17.1')]
self.assertTrue(userReqs, assertReqs)
def parse(
cls, line: str, parsed: Optional[Dict[str, Optional[str]]] = None
) -> "NamedRequirement":
r = cls()
try:
PackageRequirement.__init__(r, line)
r.marker = get_marker(r.marker)
except InvalidMarker as e:
raise RequirementError("Invalid marker: %s" % str(e)) from None
return r
def __init__(self, line, deps=None):
if deps is None:
deps = []
self.deps = [Requirement.parse(d) for d in deps]
self.req = Requirement.parse(line)
self.key = key_from_req(self.req)
self.specifier = self.req.specifier
self.version = line.split("==")[1]
def testRequirmentsString(self):
"""
testRequirmentsString will test a normal requirments list
"""
REQUIREMENTS = ['scikit-learn==0.16.1', 'pandas==0.17.1']
userReqs = parseUserRequirementsList(REQUIREMENTS)
assertReqs = [
Requirement.parse('scikit-learn==0.16.1'),
Requirement.parse('pandas==0.17.1')
]
self.assertTrue(userReqs, assertReqs)
def __init__(self, line, deps=None):
if deps is None:
deps = []
self.deps = [Requirement.parse(d) for d in deps]
self.req = Requirement.parse(line)
self.key = key_from_req(self.req)
self.specifier = self.req.specifier
self.version = line.split("==")[1]
def merge(session, explicit=""):
"""
Merges implicit and explicit requirements together. Implicit requirements
are pulled out the user's session (i.e. globals()). Explicit requirements
are provided directly by the user (e.g. ["sklearn==0.15.2"]). This
function reconciles them and merges them into one set of requirements.
Warnings are given to the user in case of version mismatche or modules
that do not need to be required explicitly.
"""
implicit_dict = {}
for r in implicit(session):
implicit_dict[r.project_name] = r
# explicit can be one requirement in a string, or many in a list.
if isinstance(explicit, basestring) and explicit:
explicit = [explicit]
explicit_list = [Requirement.parse(r) for r in explicit]
explicit_dict = {}
for r in explicit_list:
explicit_dict[r.project_name] = r
for project_name, exp_req in explicit_dict.items():
# To be polite, we keep the explicit dependencies and add the implicit
# ones to them. We respect versions on the former, except in the case
# of yhat, which should be the installed version.
if project_name in implicit_dict:
imp_req = implicit_dict[project_name]
if exp_req == imp_req:
warn(
"Dependency %s found implicitly. It can be removed "
"from REQUIREMENTS." % exp_req
)
elif project_name == "yhat":
warn(
"Dependency yhat will be set to version %s." % imp_req
)
else:
warn(
"Dependency %s specified in REQUIREMENTS, but %s is "
"installed. Using the former." % (exp_req, imp_req)
)
implicit_dict[project_name] = exp_req
else:
implicit_dict[project_name] = exp_req
# Verify that yhat is set to the installed version. Normally this whould
# show up as an implicit requirement. But we want to be extra sure.
import yhat
implicit_dict['yhat'] = Requirement.parse('yhat==%s' % yhat.__version__)
return implicit_dict.values()
def merge(session, explicit=""):
"""
Merges implicit and explicit requirements together. Implicit requirements
are pulled out the user's session (i.e. globals()). Explicit requirements
are provided directly by the user (e.g. ["sklearn==0.15.2"]). This
function reconciles them and merges them into one set of requirements.
Warnings are given to the user in case of version mismatche or modules
that do not need to be required explicitly.
"""
implicit_dict = {}
for r in implicit(session):
implicit_dict[r.project_name] = r
# explicit can be one requirement in a string, or many in a list.
if isinstance(explicit, basestring) and explicit:
explicit = [explicit]
explicit_list = [Requirement.parse(r) for r in explicit]
explicit_dict = {}
for r in explicit_list:
explicit_dict[r.project_name] = r
for project_name, exp_req in explicit_dict.items():
# To be polite, we keep the explicit dependencies and add the implicit
# ones to them. We respect versions on the former, except in the case
# of yhat, which should be the installed version.
if project_name in implicit_dict:
imp_req = implicit_dict[project_name]
if exp_req == imp_req:
warn("Dependency %s found implicitly. It can be removed "
"from REQUIREMENTS." % exp_req)
elif project_name == "yhat":
warn("Dependency yhat will be set to version %s." % imp_req)
else:
warn("Dependency %s specified in REQUIREMENTS, but %s is "
"installed. Using the former." % (exp_req, imp_req))
implicit_dict[project_name] = exp_req
else:
implicit_dict[project_name] = exp_req
# Verify that yhat is set to the installed version. Normally this whould
# show up as an implicit requirement. But we want to be extra sure.
import yhat
implicit_dict['yhat'] = Requirement.parse('yhat==%s' % yhat.__version__)
return implicit_dict.values()
def dist_requires(dist: "Distribution") -> Iterable[Requirement]:
"""Mimics pkg_resources.Distribution.requires for the case of no
extras. This doesn't fulfill that API's `extras` parameter but
satisfies the needs of pip-tools."""
reqs = (Requirement.parse(req) for req in (dist.requires or ()))
return [
req for req in reqs
if not req.marker or req.marker.evaluate({"extra": None})
]
def run(self):
for line_number, line in enumerate(self.project_requirements, start=1):
if self.is_valid_requirement(line):
requirement = Requirement.parse(line)
message = self._run(requirement, line_number)
if message:
sys.stderr.write(message)
sys.exit(self.exit_code)
def missing_requirements(self, reqs):
"""Return a list of the requirements from reqs that are not present
"""
missing = []
with self:
ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep))
for req in reqs:
try:
if ws.find(Requirement.parse(req)) is None:
missing.append(req)
except VersionConflict:
missing.append(req)
return missing
def missing_requirements(self, reqs):
"""Return a list of the requirements from reqs that are not present
"""
missing = []
with self:
ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep))
for req in reqs:
try:
if ws.find(Requirement.parse(req)) is None:
missing.append(req)
except VersionConflict:
missing.append(req)
return missing
def check_requirements(self, reqs: Iterable[str]) -> Iterable[str]:
missing = set()
conflicting = set()
if reqs:
ws = WorkingSet(self._prefix.lib_dirs)
for req in reqs:
try:
if ws.find(Requirement.parse(req)) is None:
missing.add(req)
except VersionConflict:
conflicting.add(req)
if conflicting:
raise BuildError(f"Conflicting requirements: {', '.join(conflicting)}")
return missing
def parseUserRequirementsList(reqList):
PACKAGE_LIMIT = 25
pkgCount = 0
userReqsRaw = []
userReqs = []
for r in reqList:
# Look for .txt
r = r.strip().strip('\n')
if r[-4:] == '.txt':
pkgList = []
f = open(r, 'r')
for line in f:
line = line.strip('\n').strip()
if line[0] != '#':
pkgList.append(line)
f.close()
reqList.extend(pkgList)
else:
try:
if r[:4] != 'yhat':
userReqsRaw.append(Requirement.parse(r))
pkgCount += 1
except RequirementParseError:
if r[:3] == 'git':
userReqsRaw.append(r)
pkgCount += 1
else:
print('Package ' + r + ' was not recognized as a valid package.')
except:
print("Unexpected error:", sys.exc_info()[0])
raise
if pkgCount > PACKAGE_LIMIT:
warn(
"\nYou have tried to add %s requirmements, which exceeds the maximum amount you can add during a deployment.\n"
"If possible, please consider explicily adding fewer requirements and try to re-deploy.\n"
"Or if you require this many packages, contact Yhat to upgrade your base image.\n" % str(pkgCount)
)
# block the deployment
sys.exit()
else:
for pkg in userReqsRaw:
if pkg not in userReqs:
userReqs.append(pkg)
return userReqs
def check_requirements(self, reqs):
# type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]]
"""Return 2 sets:
- conflicting requirements: set of (installed, wanted) reqs tuples
- missing requirements: set of reqs
"""
missing = set()
conflicting = set()
if reqs:
ws = WorkingSet(self._lib_dirs)
for req in reqs:
try:
if ws.find(Requirement.parse(req)) is None:
missing.add(req)
except VersionConflict as e:
conflicting.add((str(e.args[0].as_requirement()), str(e.args[1])))
return conflicting, missing
def check_requirements(self, reqs):
# type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]]
"""Return 2 sets:
- conflicting requirements: set of (installed, wanted) reqs tuples
- missing requirements: set of reqs
"""
missing = set()
conflicting = set()
if reqs:
ws = WorkingSet(self._lib_dirs)
for req in reqs:
try:
if ws.find(Requirement.parse(req)) is None:
missing.add(req)
except VersionConflict as e:
conflicting.add((str(e.args[0].as_requirement()),
str(e.args[1])))
return conflicting, missing
def build_and_install_via_wheel(self):
if not wheel_available:
raise RuntimeError('The `wheel` package is not available.')
build = self.reinitialize_command('build', reinit_subcommands=1)
build.build_base = self.bdist_base
bdist_wheel = self.reinitialize_command('bdist_wheel',
reinit_subcommands=1)
bdist_wheel.bdist_base = self.bdist_base
bdist_wheel.keep_temp = True
self.run_command('bdist_wheel')
name = self.distribution.get_name()
pip.wheel.move_wheel_files(
name=self.name,
req=WhlRequirement.parse('{}=={}'.format(name, self.version)),
wheeldir=bdist_wheel.bdist_dir,
root=self.install_dir,
prefix=self.prefix,
)
def testPipFreeze(self):
"""
testPipFreeze will test that we can parse a requirements.txt file
from pip freeze
"""
# Set the requirments to our test requirments list
REQUIREMENTS = ['./testdata/realtime/test_reqs.txt']
# Run this through the parser
userReqs = parseUserRequirementsList(REQUIREMENTS)
assertReqs = [Requirement.parse('scikit-learn==0.16.1'),
Requirement.parse('pandas==0.17.1'),
Requirement.parse('dill==0.2.5'),
Requirement.parse('terragon==0.3.0'),
Requirement.parse('progressbar2==3.10.1'),
Requirement.parse('requests'),
Requirement.parse('requests-toolbelt')]
self.assertTrue(userReqs, assertReqs)
def initializeRequirements(model):
requirements = {
'modelSpecified': [],
'required': [],
'autodetected': []
}
userReqs = getattr(model, "REQUIREMENTS", "")
if isinstance(userReqs, str):
userReqs = [r for r in userReqs.splitlines() if r]
if userReqs:
userReqs = parseUserRequirementsList(userReqs)
requirements['modelSpecified'] = userReqs
# Always add yhat package to required list with the installed version.
import yhat
yhatReq = Requirement.parse('yhat==%s' % yhat.__version__)
requirements['required'].append(yhatReq)
return requirements
def build_and_install_via_wheel(self):
if not wheel_available:
raise RuntimeError('The `wheel` package is not available.')
build = self.reinitialize_command('build', reinit_subcommands=1)
build.build_base = self.bdist_base
bdist_wheel = self.reinitialize_command(
'bdist_wheel',
reinit_subcommands=1
)
bdist_wheel.bdist_base = self.bdist_base
bdist_wheel.keep_temp = True
self.run_command('bdist_wheel')
name = self.distribution.get_name()
pip.wheel.move_wheel_files(
name=self.name,
req=WhlRequirement.parse('{}=={}'.format(name, self.version)),
wheeldir=bdist_wheel.bdist_dir,
root=self.install_dir,
prefix=self.prefix,
)
def implicit(session):
"""
Returns a list of Requirement instances for all the library dependencies
of a given session. These are matched using the contents of "top_level.txt"
metadata for all package names in the session.
"""
package_names = [_get_package_name(g) for g in session.values()]
package_names = set(filter(None, package_names))
reqs = {}
for d in get_installed_distributions():
for top_level in d._get_metadata("top_level.txt"):
if top_level in package_names:
# Sanity check: if a distribution is already in our
# requirements, make sure we only keep the latest version.
if d.project_name in reqs:
reqs[d.project_name] = max(reqs[d.project_name], d.version)
else:
reqs[d.project_name] = d.version
return [Requirement.parse('%s==%s' % r) for r in reqs.items()]
def implicit(session):
"""
Returns a list of Requirement instances for all the library dependencies
of a given session. These are matched using the contents of "top_level.txt"
metadata for all package names in the session.
"""
package_names = [_get_package_name(g) for g in session.values()]
package_names = set(filter(None, package_names))
reqs = {}
for d in get_installed_distributions():
for top_level in d._get_metadata("top_level.txt"):
if top_level in package_names:
# Sanity check: if a distribution is already in our
# requirements, make sure we only keep the latest version.
if d.project_name in reqs:
reqs[d.project_name] = max(reqs[d.project_name], d.version)
else:
reqs[d.project_name] = d.version
return [Requirement.parse('%s==%s' % r) for r in reqs.items()]
def config_logging():
global configured
if configured:
logging.info(f"Logging already configured.")
return
configured = True
log_conf_file = os.getenv('LOG_CONFIG') # None
if not log_conf_file:
log_conf_file = resource_filename(Requirement.parse('eam_core'),
"logconf.yml")
if not os.path.exists(log_conf_file):
import pathlib
log_conf_file = pathlib.Path(__file__).parent.absolute().joinpath(
"logconf.yml")
with open(log_conf_file, 'r') as f:
log_config = yaml.safe_load(f.read())
dictConfig(log_config)
logging.info(f"Configured logging from {log_conf_file}")
def testPipFreeze(self):
"""
testPipFreeze will test that we can parse a requirements.txt file
from pip freeze
"""
# Set the requirments to our test requirments list
REQUIREMENTS = ['./testdata/realtime/test_reqs.txt']
# Run this through the parser
userReqs = parseUserRequirementsList(REQUIREMENTS)
assertReqs = [
Requirement.parse('scikit-learn==0.16.1'),
Requirement.parse('pandas==0.17.1'),
Requirement.parse('dill==0.2.5'),
Requirement.parse('terragon==0.3.0'),
Requirement.parse('progressbar2==3.10.1'),
Requirement.parse('requests'),
Requirement.parse('requests-toolbelt')
]
self.assertTrue(userReqs, assertReqs)
from pytest import fixture, mark, raises
from pip._internal.network.lazy_wheel import (
HTTPRangeRequestUnsupported,
dist_from_wheel_url,
)
from pip._internal.network.session import PipSession
from tests.lib.server import file_response
MYPY_0_782_WHL = (
'https://files.pythonhosted.org/packages/9d/65/'
'b96e844150ce18b9892b155b780248955ded13a2581d31872e7daa90a503/'
'mypy-0.782-py3-none-any.whl'
)
MYPY_0_782_REQS = {
Requirement('typed-ast (<1.5.0,>=1.4.0)'),
Requirement('typing-extensions (>=3.7.4)'),
Requirement('mypy-extensions (<0.5.0,>=0.4.3)'),
Requirement('psutil (>=4.0); extra == "dmypy"'),
}
@fixture
def session():
return PipSession()
@fixture
def mypy_whl_no_range(mock_server, shared_data):
mypy_whl = shared_data.packages / 'mypy-0.782-py3-none-any.whl'
mock_server.set_responses([file_response(mypy_whl)])
self._temp_dir.cleanup()
>>>>>>> 71358189c5e72ee2ac9883b408a2f540a7f5745e
def check_requirements(self, reqs):
# type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]]
"""Return 2 sets:
- conflicting requirements: set of (installed, wanted) reqs tuples
- missing requirements: set of reqs
"""
missing = set()
conflicting = set()
if reqs:
ws = WorkingSet(self._lib_dirs)
for req in reqs:
try:
if ws.find(Requirement.parse(req)) is None:
missing.add(req)
except VersionConflict as e:
conflicting.add((str(e.args[0].as_requirement()),
str(e.args[1])))
return conflicting, missing
def install_requirements(
self,
finder, # type: PackageFinder
requirements, # type: Iterable[str]
prefix_as_string, # type: str
message # type: Optional[str]
):
# type: (...) -> None
prefix = self._prefixes[prefix_as_string]