示例#1
0
def test_dict_metadata_works():
    name = "simple"
    version = "0.1.0"
    require_a = "a==1.0"
    require_b = "b==1.1; extra == 'also_b'"
    requires = [require_a, require_b, "c==1.2; extra == 'also_c'"]
    extras = ["also_b", "also_c"]
    requires_python = ">=3"

    metadata = Message()
    metadata["Name"] = name
    metadata["Version"] = version
    for require in requires:
        metadata["Requires-Dist"] = require
    for extra in extras:
        metadata["Provides-Extra"] = extra
    metadata["Requires-Python"] = requires_python

    inner_metadata = DictMetadata({
        "METADATA": ensure_binary(metadata.as_string())
    })
    dist = DistInfoDistribution(
        location="<in-memory>", metadata=inner_metadata, project_name=name
    )

    assert name == dist.project_name
    assert version == dist.version
    assert set(extras) == set(dist.extras)
    assert [Requirement.parse(require_a)] == dist.requires([])
    assert [
        Requirement.parse(require_a), Requirement.parse(require_b)
    ] == dist.requires(["also_b"])
    assert metadata.as_string() == get_metadata(dist).as_string()
    assert requires_python == get_requires_python(dist)
 def is_valid_requirement(cls, requirement):
     try:
         Requirement.parse(requirement)
     except ValueError:
         return False
     else:
         return True
示例#3
0
def merge(requirements):
    """
    Merges autodetected and explicit requirements together. Autodetected
    requirements are pulled out the user's session (i.e. globals()).
    Explicit requirements are provided directly by the user. This
    function reconciles them and merges them into one set of requirements.
    Warnings are given to the user in case of version mismatch.
    Because we want to move away from implicitly getting requirements, we warn
    the user if there are implicitly detecdet but not explicitly stated
    requirements.
    """
    implicit_dict = {}
    for r in requirements['autodetected']:
        implicit_dict[r.project_name] = r

    explicit_dict = {}
    for r in requirements['modelSpecified']:
        if type(r) != str:
            explicit_dict[r.project_name] = r

    for project_name, exp_req in list(explicit_dict.items()):
        # To be polite, we keep the explicit dependencies and add the implicit
        # ones to them. We respect versions on the former, except in the case
        # of yhat, which should be the installed version.
        if project_name in implicit_dict:
            imp_req = implicit_dict[project_name]
            if exp_req == imp_req:
                # we only need one of these, remove the implicit, but don't need
                # to warn the user
                requirements['autodetected'].remove(Requirement.parse(str(imp_req)))
            elif project_name == "yhat":
                warn(
                    "The yhat package can be removed from REQUIREMENTS. "
                    "It is required and added for you."
                )
                try:
                    requirements['autodetected'].remove(Requirement.parse(str(imp_req)))
                    requirements['modelSpecified'].remove(Requirement.parse(str(exp_req)))
                except:
                    pass
            else:
                warn(
                    "You have explicitly %s as a requirement, which differs from %s, "
                    "which is was implicitly found to be installed locally\n"
                    "Deploying with explicitly defined package: %s " % (exp_req, imp_req, exp_req)
                )
                requirements['autodetected'].remove(Requirement.parse(str(imp_req)))

    # Loop through the implicit dict and notify users if they haven't explicitly
    # specified a requirement
    for project_name, imp_req in list(implicit_dict.items()):
        if project_name not in explicit_dict:
            warn(
                "Dependency %s was found with autodetection, but we recommend "
                "explicitly stating your requirements." % (imp_req)
            )

    return requirements
示例#4
0
 def testRequirmentsString(self):
     """
     testRequirmentsString will test a normal requirments list
     """
     REQUIREMENTS = ['scikit-learn==0.16.1', 'pandas==0.17.1']
     userReqs = parseUserRequirementsList(REQUIREMENTS)
     assertReqs = [Requirement.parse('scikit-learn==0.16.1'),
          Requirement.parse('pandas==0.17.1')]
     self.assertTrue(userReqs, assertReqs)
示例#5
0
 def parse(
     cls, line: str, parsed: Optional[Dict[str, Optional[str]]] = None
 ) -> "NamedRequirement":
     r = cls()
     try:
         PackageRequirement.__init__(r, line)
         r.marker = get_marker(r.marker)
     except InvalidMarker as e:
         raise RequirementError("Invalid marker: %s" % str(e)) from None
     return r
示例#6
0
    def __init__(self, line, deps=None):
        if deps is None:
            deps = []
        self.deps = [Requirement.parse(d) for d in deps]

        self.req = Requirement.parse(line)

        self.key = key_from_req(self.req)
        self.specifier = self.req.specifier

        self.version = line.split("==")[1]
示例#7
0
 def testRequirmentsString(self):
     """
     testRequirmentsString will test a normal requirments list
     """
     REQUIREMENTS = ['scikit-learn==0.16.1', 'pandas==0.17.1']
     userReqs = parseUserRequirementsList(REQUIREMENTS)
     assertReqs = [
         Requirement.parse('scikit-learn==0.16.1'),
         Requirement.parse('pandas==0.17.1')
     ]
     self.assertTrue(userReqs, assertReqs)
示例#8
0
    def __init__(self, line, deps=None):
        if deps is None:
            deps = []
        self.deps = [Requirement.parse(d) for d in deps]

        self.req = Requirement.parse(line)

        self.key = key_from_req(self.req)
        self.specifier = self.req.specifier

        self.version = line.split("==")[1]
示例#9
0
def merge(session, explicit=""):
    """
    Merges implicit and explicit requirements together. Implicit requirements
    are pulled out the user's session (i.e. globals()). Explicit requirements
    are provided directly by the user (e.g. ["sklearn==0.15.2"]). This
    function reconciles them and merges them into one set of requirements.
    Warnings are given to the user in case of version mismatche or modules
    that do not need to be required explicitly.
    """
    implicit_dict = {}
    for r in implicit(session):
        implicit_dict[r.project_name] = r 

    # explicit can be one requirement in a string, or many in a list.
    if isinstance(explicit, basestring) and explicit:
        explicit = [explicit]
    explicit_list = [Requirement.parse(r) for r in explicit]
    explicit_dict = {}
    for r in explicit_list:
        explicit_dict[r.project_name] = r

    for project_name, exp_req in explicit_dict.items():
        # To be polite, we keep the explicit dependencies and add the implicit
        # ones to them. We respect versions on the former, except in the case
        # of yhat, which should be the installed version.
        if project_name in implicit_dict:
            imp_req = implicit_dict[project_name]
            if exp_req == imp_req:
                warn(
                    "Dependency %s found implicitly. It can be removed "
                    "from REQUIREMENTS." % exp_req
                )

            elif project_name == "yhat":
                warn(
                    "Dependency yhat will be set to version %s." % imp_req
                )

            else:
                warn(
                    "Dependency %s specified in REQUIREMENTS, but %s is "
                    "installed. Using the former." % (exp_req, imp_req)
                )
                implicit_dict[project_name] = exp_req

        else:
            implicit_dict[project_name] = exp_req

    # Verify that yhat is set to the installed version. Normally this whould
    # show up as an implicit requirement. But we want to be extra sure.
    import yhat
    implicit_dict['yhat'] = Requirement.parse('yhat==%s' % yhat.__version__)

    return implicit_dict.values()
示例#10
0
def merge(session, explicit=""):
    """
    Merges implicit and explicit requirements together. Implicit requirements
    are pulled out the user's session (i.e. globals()). Explicit requirements
    are provided directly by the user (e.g. ["sklearn==0.15.2"]). This
    function reconciles them and merges them into one set of requirements.
    Warnings are given to the user in case of version mismatche or modules
    that do not need to be required explicitly.
    """
    implicit_dict = {}
    for r in implicit(session):
        implicit_dict[r.project_name] = r

    # explicit can be one requirement in a string, or many in a list.
    if isinstance(explicit, basestring) and explicit:
        explicit = [explicit]
    explicit_list = [Requirement.parse(r) for r in explicit]
    explicit_dict = {}
    for r in explicit_list:
        explicit_dict[r.project_name] = r

    for project_name, exp_req in explicit_dict.items():
        # To be polite, we keep the explicit dependencies and add the implicit
        # ones to them. We respect versions on the former, except in the case
        # of yhat, which should be the installed version.
        if project_name in implicit_dict:
            imp_req = implicit_dict[project_name]
            if exp_req == imp_req:
                warn("Dependency %s found implicitly. It can be removed "
                     "from REQUIREMENTS." % exp_req)

            elif project_name == "yhat":
                warn("Dependency yhat will be set to version %s." % imp_req)

            else:
                warn("Dependency %s specified in REQUIREMENTS, but %s is "
                     "installed. Using the former." % (exp_req, imp_req))
                implicit_dict[project_name] = exp_req

        else:
            implicit_dict[project_name] = exp_req

    # Verify that yhat is set to the installed version. Normally this whould
    # show up as an implicit requirement. But we want to be extra sure.
    import yhat
    implicit_dict['yhat'] = Requirement.parse('yhat==%s' % yhat.__version__)

    return implicit_dict.values()
示例#11
0
 def dist_requires(dist: "Distribution") -> Iterable[Requirement]:
     """Mimics pkg_resources.Distribution.requires for the case of no
     extras. This doesn't fulfill that API's `extras` parameter but
     satisfies the needs of pip-tools."""
     reqs = (Requirement.parse(req) for req in (dist.requires or ()))
     return [
         req for req in reqs
         if not req.marker or req.marker.evaluate({"extra": None})
     ]
    def run(self):
        for line_number, line in enumerate(self.project_requirements, start=1):
            if self.is_valid_requirement(line):
                requirement = Requirement.parse(line)
                message = self._run(requirement, line_number)
                if message:
                    sys.stderr.write(message)

        sys.exit(self.exit_code)
示例#13
0
 def missing_requirements(self, reqs):
     """Return a list of the requirements from reqs that are not present
     """
     missing = []
     with self:
         ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep))
         for req in reqs:
             try:
                 if ws.find(Requirement.parse(req)) is None:
                     missing.append(req)
             except VersionConflict:
                 missing.append(req)
         return missing
示例#14
0
 def missing_requirements(self, reqs):
     """Return a list of the requirements from reqs that are not present
     """
     missing = []
     with self:
         ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep))
         for req in reqs:
             try:
                 if ws.find(Requirement.parse(req)) is None:
                     missing.append(req)
             except VersionConflict:
                 missing.append(req)
         return missing
示例#15
0
 def check_requirements(self, reqs: Iterable[str]) -> Iterable[str]:
     missing = set()
     conflicting = set()
     if reqs:
         ws = WorkingSet(self._prefix.lib_dirs)
         for req in reqs:
             try:
                 if ws.find(Requirement.parse(req)) is None:
                     missing.add(req)
             except VersionConflict:
                 conflicting.add(req)
     if conflicting:
         raise BuildError(f"Conflicting requirements: {', '.join(conflicting)}")
     return missing
示例#16
0
def parseUserRequirementsList(reqList):
    PACKAGE_LIMIT = 25
    pkgCount = 0
    userReqsRaw = []
    userReqs = []
    for r in reqList:
        # Look for .txt
        r = r.strip().strip('\n')
        if r[-4:] == '.txt':
            pkgList = []
            f = open(r, 'r')
            for line in f:
                line = line.strip('\n').strip()
                if line[0] != '#':
                    pkgList.append(line)
            f.close()
            reqList.extend(pkgList)
        else:
            try:
                if r[:4] != 'yhat':
                    userReqsRaw.append(Requirement.parse(r))
                    pkgCount += 1
            except RequirementParseError:
                if r[:3] == 'git':
                    userReqsRaw.append(r)
                    pkgCount += 1
                else:
                    print('Package ' + r + ' was not recognized as a valid package.')
            except:
                print("Unexpected error:", sys.exc_info()[0])
                raise

    if pkgCount > PACKAGE_LIMIT:
        warn(
            "\nYou have tried to add %s requirmements, which exceeds the maximum amount you can add during a deployment.\n"
            "If possible, please consider explicily adding fewer requirements and try to re-deploy.\n"
            "Or if you require this many packages, contact Yhat to upgrade your base image.\n" % str(pkgCount)
        )
        # block the deployment
        sys.exit()
    else:
        for pkg in userReqsRaw:
            if pkg not in userReqs:
                userReqs.append(pkg)

    return userReqs
示例#17
0
 def check_requirements(self, reqs):
     # type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]]
     """Return 2 sets:
         - conflicting requirements: set of (installed, wanted) reqs tuples
         - missing requirements: set of reqs
     """
     missing = set()
     conflicting = set()
     if reqs:
         ws = WorkingSet(self._lib_dirs)
         for req in reqs:
             try:
                 if ws.find(Requirement.parse(req)) is None:
                     missing.add(req)
             except VersionConflict as e:
                 conflicting.add((str(e.args[0].as_requirement()), str(e.args[1])))
     return conflicting, missing
示例#18
0
 def check_requirements(self, reqs):
     # type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]]
     """Return 2 sets:
         - conflicting requirements: set of (installed, wanted) reqs tuples
         - missing requirements: set of reqs
     """
     missing = set()
     conflicting = set()
     if reqs:
         ws = WorkingSet(self._lib_dirs)
         for req in reqs:
             try:
                 if ws.find(Requirement.parse(req)) is None:
                     missing.add(req)
             except VersionConflict as e:
                 conflicting.add((str(e.args[0].as_requirement()),
                                  str(e.args[1])))
     return conflicting, missing
示例#19
0
 def build_and_install_via_wheel(self):
     if not wheel_available:
         raise RuntimeError('The `wheel` package is not available.')
     build = self.reinitialize_command('build', reinit_subcommands=1)
     build.build_base = self.bdist_base
     bdist_wheel = self.reinitialize_command('bdist_wheel',
                                             reinit_subcommands=1)
     bdist_wheel.bdist_base = self.bdist_base
     bdist_wheel.keep_temp = True
     self.run_command('bdist_wheel')
     name = self.distribution.get_name()
     pip.wheel.move_wheel_files(
         name=self.name,
         req=WhlRequirement.parse('{}=={}'.format(name, self.version)),
         wheeldir=bdist_wheel.bdist_dir,
         root=self.install_dir,
         prefix=self.prefix,
     )
示例#20
0
 def testPipFreeze(self):
     """
     testPipFreeze will test that we can parse a requirements.txt file
     from pip freeze
     """
     # Set the requirments to our test requirments list
     REQUIREMENTS = ['./testdata/realtime/test_reqs.txt']
     # Run this through the parser
     userReqs = parseUserRequirementsList(REQUIREMENTS)
     assertReqs = [Requirement.parse('scikit-learn==0.16.1'),
          Requirement.parse('pandas==0.17.1'),
          Requirement.parse('dill==0.2.5'),
          Requirement.parse('terragon==0.3.0'),
          Requirement.parse('progressbar2==3.10.1'),
          Requirement.parse('requests'),
          Requirement.parse('requests-toolbelt')]
     self.assertTrue(userReqs, assertReqs)
示例#21
0
def initializeRequirements(model):
    requirements = {
        'modelSpecified': [],
        'required': [],
        'autodetected': []
    }
    userReqs = getattr(model, "REQUIREMENTS", "")
    if isinstance(userReqs, str):
        userReqs = [r for r in userReqs.splitlines() if r]
    if userReqs:
        userReqs = parseUserRequirementsList(userReqs)
        requirements['modelSpecified'] = userReqs

    # Always add yhat package to required list with the installed version.
    import yhat
    yhatReq = Requirement.parse('yhat==%s' % yhat.__version__)
    requirements['required'].append(yhatReq)

    return requirements
示例#22
0
 def build_and_install_via_wheel(self):
     if not wheel_available:
         raise RuntimeError('The `wheel` package is not available.')
     build = self.reinitialize_command('build', reinit_subcommands=1)
     build.build_base = self.bdist_base
     bdist_wheel = self.reinitialize_command(
         'bdist_wheel',
         reinit_subcommands=1
     )
     bdist_wheel.bdist_base = self.bdist_base
     bdist_wheel.keep_temp = True
     self.run_command('bdist_wheel')
     name = self.distribution.get_name()
     pip.wheel.move_wheel_files(
         name=self.name,
         req=WhlRequirement.parse('{}=={}'.format(name, self.version)),
         wheeldir=bdist_wheel.bdist_dir,
         root=self.install_dir,
         prefix=self.prefix,
     )
示例#23
0
def implicit(session):
    """
    Returns a list of Requirement instances for all the library dependencies
    of a given session. These are matched using the contents of "top_level.txt"
    metadata for all package names in the session.
    """
    package_names = [_get_package_name(g) for g in session.values()]
    package_names = set(filter(None, package_names))

    reqs = {}
    for d in get_installed_distributions():
        for top_level in d._get_metadata("top_level.txt"):
            if top_level in package_names:
                # Sanity check: if a distribution is already in our
                # requirements, make sure we only keep the latest version.
                if d.project_name in reqs:
                    reqs[d.project_name] = max(reqs[d.project_name], d.version)
                else:
                    reqs[d.project_name] = d.version

    return [Requirement.parse('%s==%s' % r) for r in reqs.items()]
示例#24
0
def implicit(session):
    """
    Returns a list of Requirement instances for all the library dependencies
    of a given session. These are matched using the contents of "top_level.txt"
    metadata for all package names in the session.
    """
    package_names = [_get_package_name(g) for g in session.values()]
    package_names = set(filter(None, package_names))

    reqs = {}
    for d in get_installed_distributions():
        for top_level in d._get_metadata("top_level.txt"):
            if top_level in package_names:
                # Sanity check: if a distribution is already in our
                # requirements, make sure we only keep the latest version.
                if d.project_name in reqs:
                    reqs[d.project_name] = max(reqs[d.project_name], d.version)
                else:
                    reqs[d.project_name] = d.version

    return [Requirement.parse('%s==%s' % r) for r in reqs.items()]
示例#25
0
def config_logging():
    global configured
    if configured:
        logging.info(f"Logging already configured.")
        return
    configured = True

    log_conf_file = os.getenv('LOG_CONFIG')  # None

    if not log_conf_file:
        log_conf_file = resource_filename(Requirement.parse('eam_core'),
                                          "logconf.yml")
        if not os.path.exists(log_conf_file):
            import pathlib
            log_conf_file = pathlib.Path(__file__).parent.absolute().joinpath(
                "logconf.yml")

    with open(log_conf_file, 'r') as f:
        log_config = yaml.safe_load(f.read())
    dictConfig(log_config)
    logging.info(f"Configured logging from {log_conf_file}")
示例#26
0
 def testPipFreeze(self):
     """
     testPipFreeze will test that we can parse a requirements.txt file
     from pip freeze
     """
     # Set the requirments to our test requirments list
     REQUIREMENTS = ['./testdata/realtime/test_reqs.txt']
     # Run this through the parser
     userReqs = parseUserRequirementsList(REQUIREMENTS)
     assertReqs = [
         Requirement.parse('scikit-learn==0.16.1'),
         Requirement.parse('pandas==0.17.1'),
         Requirement.parse('dill==0.2.5'),
         Requirement.parse('terragon==0.3.0'),
         Requirement.parse('progressbar2==3.10.1'),
         Requirement.parse('requests'),
         Requirement.parse('requests-toolbelt')
     ]
     self.assertTrue(userReqs, assertReqs)
示例#27
0
from pytest import fixture, mark, raises

from pip._internal.network.lazy_wheel import (
    HTTPRangeRequestUnsupported,
    dist_from_wheel_url,
)
from pip._internal.network.session import PipSession
from tests.lib.server import file_response

MYPY_0_782_WHL = (
    'https://files.pythonhosted.org/packages/9d/65/'
    'b96e844150ce18b9892b155b780248955ded13a2581d31872e7daa90a503/'
    'mypy-0.782-py3-none-any.whl'
)
MYPY_0_782_REQS = {
    Requirement('typed-ast (<1.5.0,>=1.4.0)'),
    Requirement('typing-extensions (>=3.7.4)'),
    Requirement('mypy-extensions (<0.5.0,>=0.4.3)'),
    Requirement('psutil (>=4.0); extra == "dmypy"'),
}


@fixture
def session():
    return PipSession()


@fixture
def mypy_whl_no_range(mock_server, shared_data):
    mypy_whl = shared_data.packages / 'mypy-0.782-py3-none-any.whl'
    mock_server.set_responses([file_response(mypy_whl)])
示例#28
0
        self._temp_dir.cleanup()

>>>>>>> 71358189c5e72ee2ac9883b408a2f540a7f5745e
    def check_requirements(self, reqs):
        # type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]]
        """Return 2 sets:
            - conflicting requirements: set of (installed, wanted) reqs tuples
            - missing requirements: set of reqs
        """
        missing = set()
        conflicting = set()
        if reqs:
            ws = WorkingSet(self._lib_dirs)
            for req in reqs:
                try:
                    if ws.find(Requirement.parse(req)) is None:
                        missing.add(req)
                except VersionConflict as e:
                    conflicting.add((str(e.args[0].as_requirement()),
                                     str(e.args[1])))
        return conflicting, missing

    def install_requirements(
        self,
        finder,  # type: PackageFinder
        requirements,  # type: Iterable[str]
        prefix_as_string,  # type: str
        message  # type: Optional[str]
    ):
        # type: (...) -> None
        prefix = self._prefixes[prefix_as_string]