Example #1
0
    def _hash_comparison(self):
        """
        Return a comparison of actual and expected hash values.

        Example::

               Expected sha256 abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde
                            or 123451234512345123451234512345123451234512345
                    Got        bcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdef

        """
        def hash_then_or(hash_name):
            # For now, all the decent hashes have 6-char names, so we can get
            # away with hard-coding space literals.
            return chain([hash_name], repeat('    or'))

        lines = []
        for hash_name, expecteds in iteritems(self.allowed):
            prefix = hash_then_or(hash_name)
            lines.extend(('        Expected %s %s' % (next(prefix), e))
                         for e in expecteds)
            lines.append('             Got        %s\n' %
                         self.gots[hash_name].hexdigest())
            prefix = '    or'
        return '\n'.join(lines)
    def __and__(self, other):
        # type: (Hashes) -> Hashes
        if not isinstance(other, Hashes):
            return NotImplemented

        # If either of the Hashes object is entirely empty (i.e. no hash
        # specified at all), all hashes from the other object are allowed.
        if not other:
            return self
        if not self:
            return other

        # Otherwise only hashes that present in both objects are allowed.
        new = {}
        for alg, values in iteritems(other._allowed):
            if alg not in self._allowed:
                continue
            new[alg] = [v for v in values if v in self._allowed[alg]]
        return Hashes(new)
Example #3
0
    def check_against_chunks(self, chunks):
        """Check good hashes against ones built from iterable of chunks of
        data.

        Raise HashMismatch if none match.

        """
        gots = {}
        for hash_name in iterkeys(self._allowed):
            try:
                gots[hash_name] = hashlib.new(hash_name)
            except (ValueError, TypeError):
                raise InstallationError('Unknown hash name: %s' % hash_name)

        for chunk in chunks:
            for hash in itervalues(gots):
                hash.update(chunk)

        for hash_name, got in iteritems(gots):
            if got.hexdigest() in self._allowed[hash_name]:
                return
        self._raise(gots)
Example #4
0
    def check_against_chunks(self, chunks):
        """Check good hashes against ones built from iterable of chunks of
        data.

        Raise HashMismatch if none match.

        """
        gots = {}
        for hash_name in iterkeys(self._allowed):
            try:
                gots[hash_name] = hashlib.new(hash_name)
            except (ValueError, TypeError):
                raise InstallationError('Unknown hash name: %s' % hash_name)

        for chunk in chunks:
            for hash in itervalues(gots):
                hash.update(chunk)

        for hash_name, got in iteritems(gots):
            if got.hexdigest() in self._allowed[hash_name]:
                return
        self._raise(gots)
Example #5
0
    def mutate(self, record):
        pairs = []

        for key, value in six.iteritems(record.__dict__):
            if key not in RESERVED_KEYWORDS:
                pairs.append([key, value])

        if pairs:
            pairs.sort(key=lambda k: k[0])

        if pairs:
            record.msg += " " + self.DELIMITER.join("{}=%s".format(key)
                                                    for key, _ in pairs)

            acc = tuple(v for _, v in pairs)
            if not isinstance(record.args, tuple):
                record.args = (record.args, ) + acc
            else:
                record.args += acc

            record.args = tuple(map(to_string, record.args))

        return record
Example #6
0
    def _hash_comparison(self):
        """
        Return a comparison of actual and expected hash values.

        Example::

               Expected sha256 abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde
                            or 123451234512345123451234512345123451234512345
                    Got        bcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdef

        """
        def hash_then_or(hash_name):
            # For now, all the decent hashes have 6-char names, so we can get
            # away with hard-coding space literals.
            return chain([hash_name], repeat('    or'))

        lines = []
        for hash_name, expecteds in iteritems(self.allowed):
            prefix = hash_then_or(hash_name)
            lines.extend(('        Expected {} {}'.format(next(prefix), e))
                         for e in expecteds)
            lines.append('             Got        {}\n'.format(
                self.gots[hash_name].hexdigest()))
        return '\n'.join(lines)
Example #7
0
def freeze(requirement=None,
           find_links=None,
           local_only=None,
           user_only=None,
           skip_regex=None,
           isolated=False,
           wheel_cache=None,
           exclude_editable=False,
           skip=()):
    find_links = find_links or []
    skip_match = None

    if skip_regex:
        skip_match = re.compile(skip_regex).search

    dependency_links = []

    for dist in pkg_resources.working_set:
        if dist.has_metadata('dependency_links.txt'):
            dependency_links.extend(
                dist.get_metadata_lines('dependency_links.txt'))
    for link in find_links:
        if '#egg=' in link:
            dependency_links.append(link)
    for link in find_links:
        yield '-f %s' % link
    installations = {}
    for dist in get_installed_distributions(local_only=local_only,
                                            skip=(),
                                            user_only=user_only):
        try:
            req = FrozenRequirement.from_dist(dist, dependency_links)
        except RequirementParseError:
            logger.warning("Could not parse requirement: %s",
                           dist.project_name)
            continue
        if exclude_editable and req.editable:
            continue
        installations[req.name] = req

    if requirement:
        # the options that don't get turned into an InstallRequirement
        # should only be emitted once, even if the same option is in multiple
        # requirements files, so we need to keep track of what has been emitted
        # so that we don't emit it again if it's seen again
        emitted_options = set()
        # keep track of which files a requirement is in so that we can
        # give an accurate warning if a requirement appears multiple times.
        req_files = collections.defaultdict(list)
        for req_file_path in requirement:
            with open(req_file_path) as req_file:
                for line in req_file:
                    if (not line.strip() or line.strip().startswith('#')
                            or (skip_match and skip_match(line))
                            or line.startswith(
                                ('-r', '--requirement', '-Z', '--always-unzip',
                                 '-f', '--find-links', '-i', '--index-url',
                                 '--pre', '--trusted-host',
                                 '--process-dependency-links',
                                 '--extra-index-url'))):
                        line = line.rstrip()
                        if line not in emitted_options:
                            emitted_options.add(line)
                            yield line
                        continue

                    if line.startswith('-e') or line.startswith('--editable'):
                        if line.startswith('-e'):
                            line = line[2:].strip()
                        else:
                            line = line[len('--editable'):].strip().lstrip('=')
                        line_req = InstallRequirement.from_editable(
                            line,
                            isolated=isolated,
                            wheel_cache=wheel_cache,
                        )
                    else:
                        line_req = InstallRequirement.from_line(
                            COMMENT_RE.sub('', line).strip(),
                            isolated=isolated,
                            wheel_cache=wheel_cache,
                        )

                    if not line_req.name:
                        logger.info(
                            "Skipping line in requirement File [%s] because "
                            "it's not clear what it would install: %s",
                            req_file_path,
                            line.strip(),
                        )
                        logger.info(
                            "  (add #egg=PackageName to the URL to avoid"
                            " this warning)")
                    elif line_req.name not in installations:
                        # either it's not installed, or it is installed
                        # but has been processed already
                        if not req_files[line_req.name]:
                            logger.warning(
                                "Requirement File [%s] contains %s, but that "
                                "package is not installed",
                                req_file_path,
                                COMMENT_RE.sub('', line).strip(),
                            )
                        else:
                            req_files[line_req.name].append(req_file_path)
                    else:
                        yield str(installations[line_req.name]).rstrip()
                        del installations[line_req.name]
                        req_files[line_req.name].append(req_file_path)

        # Warn about requirements that were included multiple times (in a
        # single requirements File or in different requirements files).
        for name, files in six.iteritems(req_files):
            if len(files) > 1:
                logger.warning("Requirement %s included multiple times [%s]",
                               name, ', '.join(sorted(set(files))))

        yield ('## The following requirements were added by ' 'pip freeze:')
    for installation in sorted(installations.values(),
                               key=lambda x: x.name.lower()):
        if canonicalize_name(installation.name) not in skip:
            yield str(installation).rstrip()
Example #8
0
def freeze(
        requirement=None,  # type: Optional[List[str]]
        find_links=None,  # type: Optional[List[str]]
        local_only=False,  # type: bool
        user_only=False,  # type: bool
        paths=None,  # type: Optional[List[str]]
        isolated=False,  # type: bool
        wheel_cache=None,  # type: Optional[WheelCache]
        exclude_editable=False,  # type: bool
        skip=()  # type: Container[str]
):
    # type: (...) -> Iterator[str]
    find_links = find_links or []

    for link in find_links:
        yield '-f {}'.format(link)
    installations = {}  # type: Dict[str, FrozenRequirement]

    for dist in get_installed_distributions(local_only=local_only,
                                            skip=(),
                                            user_only=user_only,
                                            paths=paths):
        try:
            req = FrozenRequirement.from_dist(dist)
        except RequirementParseError as exc:
            # We include dist rather than dist.project_name because the
            # dist string includes more information, like the version and
            # location. We also include the exception message to aid
            # troubleshooting.
            logger.warning(
                'Could not generate requirement for distribution %r: %s', dist,
                exc)
            continue
        if exclude_editable and req.editable:
            continue
        installations[req.canonical_name] = req

    if requirement:
        # the options that don't get turned into an InstallRequirement
        # should only be emitted once, even if the same option is in multiple
        # requirements files, so we need to keep track of what has been emitted
        # so that we don't emit it again if it's seen again
        emitted_options = set()  # type: Set[str]
        # keep track of which files a requirement is in so that we can
        # give an accurate warning if a requirement appears multiple times.
        req_files = collections.defaultdict(list)  # type: Dict[str, List[str]]
        for req_file_path in requirement:
            with open(req_file_path) as req_file:
                for line in req_file:
                    if (not line.strip() or line.strip().startswith('#')
                            or line.startswith(
                                ('-r', '--requirement', '-f', '--find-links',
                                 '-i', '--index-url', '--pre',
                                 '--trusted-host',
                                 '--process-dependency-links',
                                 '--extra-index-url', '--use-feature'))):
                        line = line.rstrip()
                        if line not in emitted_options:
                            emitted_options.add(line)
                            yield line
                        continue

                    if line.startswith('-e') or line.startswith('--editable'):
                        if line.startswith('-e'):
                            line = line[2:].strip()
                        else:
                            line = line[len('--editable'):].strip().lstrip('=')
                        line_req = install_req_from_editable(
                            line,
                            isolated=isolated,
                        )
                    else:
                        line_req = install_req_from_line(
                            COMMENT_RE.sub('', line).strip(),
                            isolated=isolated,
                        )

                    if not line_req.name:
                        logger.info(
                            "Skipping line in requirement file [%s] because "
                            "it's not clear what it would install: %s",
                            req_file_path,
                            line.strip(),
                        )
                        logger.info(
                            "  (add #egg=PackageName to the URL to avoid"
                            " this warning)")
                    else:
                        line_req_canonical_name = canonicalize_name(
                            line_req.name)
                        if line_req_canonical_name not in installations:
                            # either it's not installed, or it is installed
                            # but has been processed already
                            if not req_files[line_req.name]:
                                logger.warning(
                                    "Requirement file [%s] contains %s, but "
                                    "package %r is not installed",
                                    req_file_path,
                                    COMMENT_RE.sub('', line).strip(),
                                    line_req.name)
                            else:
                                req_files[line_req.name].append(req_file_path)
                        else:
                            yield str(installations[line_req_canonical_name]
                                      ).rstrip()
                            del installations[line_req_canonical_name]
                            req_files[line_req.name].append(req_file_path)

        # Warn about requirements that were included multiple times (in a
        # single requirements file or in different requirements files).
        for name, files in six.iteritems(req_files):
            if len(files) > 1:
                logger.warning("Requirement %s included multiple times [%s]",
                               name, ', '.join(sorted(set(files))))

        yield ('## The following requirements were added by ' 'pip freeze:')
    for installation in sorted(installations.values(),
                               key=lambda x: x.name.lower()):
        if installation.canonical_name not in skip:
            yield str(installation).rstrip()
Example #9
0
def freeze(
    requirement=None,  # type: Optional[List[str]]
    find_links=None,  # type: Optional[List[str]]
    local_only=None,  # type: Optional[bool]
    user_only=None,  # type: Optional[bool]
    skip_regex=None,  # type: Optional[str]
    isolated=False,  # type: bool
    wheel_cache=None,  # type: Optional[WheelCache]
    exclude_editable=False,  # type: bool
    skip=()  # type: Container[str]
):
    # type: (...) -> Iterator[str]
    find_links = find_links or []
    skip_match = None

    if skip_regex:
        skip_match = re.compile(skip_regex).search

    for link in find_links:
        yield '-f %s' % link
    installations = {}  # type: Dict[str, FrozenRequirement]
    for dist in get_installed_distributions(local_only=local_only,
                                            skip=(),
                                            user_only=user_only):
        try:
            req = FrozenRequirement.from_dist(dist)
        except RequirementParseError:
            logger.warning(
                "Could not parse requirement: %s",
                dist.project_name
            )
            continue
        if exclude_editable and req.editable:
            continue
        installations[req.name] = req

    if requirement:
        # the options that don't get turned into an InstallRequirement
        # should only be emitted once, even if the same option is in multiple
        # requirements files, so we need to keep track of what has been emitted
        # so that we don't emit it again if it's seen again
        emitted_options = set()  # type: Set[str]
        # keep track of which files a requirement is in so that we can
        # give an accurate warning if a requirement appears multiple times.
        req_files = collections.defaultdict(list)  # type: Dict[str, List[str]]
        for req_file_path in requirement:
            with open(req_file_path) as req_file:
                for line in req_file:
                    if (not line.strip() or
                            line.strip().startswith('#') or
                            (skip_match and skip_match(line)) or
                            line.startswith((
                                '-r', '--requirement',
                                '-Z', '--always-unzip',
                                '-f', '--find-links',
                                '-i', '--index-url',
                                '--pre',
                                '--trusted-host',
                                '--process-dependency-links',
                                '--extra-index-url'))):
                        line = line.rstrip()
                        if line not in emitted_options:
                            emitted_options.add(line)
                            yield line
                        continue

                    if line.startswith('-e') or line.startswith('--editable'):
                        if line.startswith('-e'):
                            line = line[2:].strip()
                        else:
                            line = line[len('--editable'):].strip().lstrip('=')
                        line_req = install_req_from_editable(
                            line,
                            isolated=isolated,
                            wheel_cache=wheel_cache,
                        )
                    else:
                        line_req = install_req_from_line(
                            COMMENT_RE.sub('', line).strip(),
                            isolated=isolated,
                            wheel_cache=wheel_cache,
                        )

                    if not line_req.name:
                        logger.info(
                            "Skipping line in requirement file [%s] because "
                            "it's not clear what it would install: %s",
                            req_file_path, line.strip(),
                        )
                        logger.info(
                            "  (add #egg=PackageName to the URL to avoid"
                            " this warning)"
                        )
                    elif line_req.name not in installations:
                        # either it's not installed, or it is installed
                        # but has been processed already
                        if not req_files[line_req.name]:
                            logger.warning(
                                "Requirement file [%s] contains %s, but "
                                "package %r is not installed",
                                req_file_path,
                                COMMENT_RE.sub('', line).strip(), line_req.name
                            )
                        else:
                            req_files[line_req.name].append(req_file_path)
                    else:
                        yield str(installations[line_req.name]).rstrip()
                        del installations[line_req.name]
                        req_files[line_req.name].append(req_file_path)

        # Warn about requirements that were included multiple times (in a
        # single requirements file or in different requirements files).
        for name, files in six.iteritems(req_files):
            if len(files) > 1:
                logger.warning("Requirement %s included multiple times [%s]",
                               name, ', '.join(sorted(set(files))))

        yield(
            '## The following requirements were added by '
            'pip freeze:'
        )
    for installation in sorted(
            installations.values(), key=lambda x: x.name.lower()):
        if canonicalize_name(installation.name) not in skip:
            yield str(installation).rstrip()
Example #10
0
            try:
                gots[hash_name] = hashlib.new(hash_name)
            except (ValueError, TypeError):
<<<<<<< HEAD
                raise InstallationError('Unknown hash name: %s' % hash_name)
=======
                raise InstallationError(
                    'Unknown hash name: {}'.format(hash_name)
                )
>>>>>>> e585743114c1741ec20dc76010f96171f3516589

        for chunk in chunks:
            for hash in itervalues(gots):
                hash.update(chunk)

        for hash_name, got in iteritems(gots):
            if got.hexdigest() in self._allowed[hash_name]:
                return
        self._raise(gots)

    def _raise(self, gots):
        # type: (Dict[str, _Hash]) -> NoReturn
        raise HashMismatch(self._allowed, gots)

    def check_against_file(self, file):
        # type: (BinaryIO) -> None
        """Check good hashes against a file-like object

        Raise HashMismatch if none match.

        """
Example #11
0
def freeze(
        requirement=None,  # type: Optional[List[str]]
        find_links=None,  # type: Optional[List[str]]
        local_only=None,  # type: Optional[bool]
        user_only=None,  # type: Optional[bool]
        skip_regex=None,  # type: Optional[str]
        isolated=False,  # type: bool
        wheel_cache=None,  # type: Optional[WheelCache]
        exclude_editable=False,  # type: bool
        skip=(),  # type: Container[str]
):
    # type: (...) -> Iterator[str]
    find_links = find_links or []
    skip_match = None

    if skip_regex:
        skip_match = re.compile(skip_regex).search

    for link in find_links:
        yield "-f %s" % link
    installations = {}  # type: Dict[str, FrozenRequirement]
    for dist in get_installed_distributions(local_only=local_only,
                                            skip=(),
                                            user_only=user_only):
        try:
            req = FrozenRequirement.from_dist(dist)
        except RequirementParseError:
            logger.warning("Could not parse requirement: %s",
                           dist.project_name)
            continue
        if exclude_editable and req.editable:
            continue
        installations[req.name] = req

    if requirement:
        # the options that don't get turned into an InstallRequirement
        # should only be emitted once, even if the same option is in multiple
        # requirements files, so we need to keep track of what has been emitted
        # so that we don't emit it again if it's seen again
        emitted_options = set()  # type: Set[str]
        # keep track of which files a requirement is in so that we can
        # give an accurate warning if a requirement appears multiple times.
        req_files = collections.defaultdict(list)  # type: Dict[str, List[str]]
        for req_file_path in requirement:
            with open(req_file_path) as req_file:
                for line in req_file:
                    if (not line.strip() or line.strip().startswith("#")
                            or (skip_match and skip_match(line))
                            or line.startswith((
                                "-r",
                                "--requirement",
                                "-Z",
                                "--always-unzip",
                                "-f",
                                "--find-links",
                                "-i",
                                "--index-url",
                                "--pre",
                                "--trusted-host",
                                "--process-dependency-links",
                                "--extra-index-url",
                            ))):
                        line = line.rstrip()
                        if line not in emitted_options:
                            emitted_options.add(line)
                            yield line
                        continue

                    if line.startswith("-e") or line.startswith("--editable"):
                        if line.startswith("-e"):
                            line = line[2:].strip()
                        else:
                            line = line[len("--editable"):].strip().lstrip("=")
                        line_req = install_req_from_editable(
                            line, isolated=isolated, wheel_cache=wheel_cache)
                    else:
                        line_req = install_req_from_line(
                            COMMENT_RE.sub("", line).strip(),
                            isolated=isolated,
                            wheel_cache=wheel_cache,
                        )

                    if not line_req.name:
                        logger.info(
                            "Skipping line in requirement file [%s] because "
                            "it's not clear what it would install: %s",
                            req_file_path,
                            line.strip(),
                        )
                        logger.info(
                            "  (add #egg=PackageName to the URL to avoid"
                            " this warning)")
                    elif line_req.name not in installations:
                        # either it's not installed, or it is installed
                        # but has been processed already
                        if not req_files[line_req.name]:
                            logger.warning(
                                "Requirement file [%s] contains %s, but "
                                "package %r is not installed",
                                req_file_path,
                                COMMENT_RE.sub("", line).strip(),
                                line_req.name,
                            )
                        else:
                            req_files[line_req.name].append(req_file_path)
                    else:
                        yield str(installations[line_req.name]).rstrip()
                        del installations[line_req.name]
                        req_files[line_req.name].append(req_file_path)

        # Warn about requirements that were included multiple times (in a
        # single requirements file or in different requirements files).
        for name, files in six.iteritems(req_files):
            if len(files) > 1:
                logger.warning(
                    "Requirement %s included multiple times [%s]",
                    name,
                    ", ".join(sorted(set(files))),
                )

        yield ("## The following requirements were added by " "pip freeze:")
    for installation in sorted(installations.values(),
                               key=lambda x: x.name.lower()):
        if canonicalize_name(installation.name) not in skip:
            yield str(installation).rstrip()
Example #12
0
        """
        def hash_then_or(hash_name):
<<<<<<< HEAD
            # type: (str) -> chain[str]
=======
>>>>>>> b66a76afa15ab74019740676a52a071b85ed8f71
            # For now, all the decent hashes have 6-char names, so we can get
            # away with hard-coding space literals.
            return chain([hash_name], repeat('    or'))

<<<<<<< HEAD
        lines = []  # type: List[str]
=======
        lines = []
>>>>>>> b66a76afa15ab74019740676a52a071b85ed8f71
        for hash_name, expecteds in iteritems(self.allowed):
            prefix = hash_then_or(hash_name)
            lines.extend(('        Expected {} {}'.format(next(prefix), e))
                         for e in expecteds)
            lines.append('             Got        {}\n'.format(
                         self.gots[hash_name].hexdigest()))
        return '\n'.join(lines)


class UnsupportedPythonVersion(InstallationError):
    """Unsupported python version according to Requires-Python package
    metadata."""


class ConfigurationFileCouldNotBeLoaded(ConfigurationError):
    """When there are errors while loading a configuration file
Example #13
0
def make_files(files):
    # type: (Dict[str, AnyStr]) -> List[File]
    return [
        File(name, ensure_binary(contents))
        for name, contents in iteritems(files)
    ]
Example #14
0
                                    "package %r is not installed",
                                    req_file_path,
                                    COMMENT_RE.sub('', line).strip(),
                                    line_req.name
                                )
                            else:
                                req_files[line_req.name].append(req_file_path)
                        else:
                            yield str(installations[
                                line_req_canonical_name]).rstrip()
                            del installations[line_req_canonical_name]
                            req_files[line_req.name].append(req_file_path)

        # Warn about requirements that were included multiple times (in a
        # single requirements file or in different requirements files).
        for name, files in six.iteritems(req_files):
            if len(files) > 1:
                logger.warning("Requirement %s included multiple times [%s]",
                               name, ', '.join(sorted(set(files))))

        yield(
            '## The following requirements were added by '
            'pip freeze:'
        )
    for installation in sorted(
            installations.values(), key=lambda x: x.name.lower()):
        if installation.canonical_name not in skip:
            yield str(installation).rstrip()


def get_requirement_info(dist):
Example #15
0
    def addContext(self, data):
        self._check_data()

        for key, value in six.iteritems(data):
            self._context.data[key] = value