Exemplo n.º 1
0
    def from_archive(cls, path):
        """
        Create and return (but do not save) an instance of a Python package object from the package
        itself.

        Twine is smart enough to crack open a tarball, zip, or wheel and parse the metadata that is
        contained in the package.

        :param path: path to the package
        :type  path: basestring
        :return: instance of a package
        :rtype:  pulp_python.plugins.models.Package
        """
        meta_dict = PackageFile.from_filename(
            path, comment='').metadata_dictionary()
        filtered_dict = {}
        # Use only a subset of the attributes in the metadata to keep package minimal.
        for key, value in meta_dict.iteritems():
            if key in PACKAGE_ATTRS:
                filtered_dict[key] = value
        filtered_dict['filename'] = os.path.basename(path)
        filtered_dict['packagetype'] = meta_dict['filetype']
        package = cls(**filtered_dict)
        package._checksum = package.checksum(path)
        package._checksum_type = CHECKSUM_TYPE
        return package
Exemplo n.º 2
0
    def from_archive(cls, path):
        """
        Create and return (but do not save) an instance of a Python package object from the package
        itself.

        Twine is smart enough to crack open a tarball, zip, or wheel and parse the metadata that is
        contained in the package.

        :param path: path to the package
        :type  path: basestring
        :return: instance of a package
        :rtype:  pulp_python.plugins.models.Package
        """
        meta_dict = PackageFile.from_filename(path, comment="").metadata_dictionary()
        filtered_dict = {}
        # Use only a subset of the attributes in the metadata to keep package minimal.
        for key, value in meta_dict.iteritems():
            if key in PACKAGE_ATTRS:
                filtered_dict[key] = value
        filtered_dict["filename"] = os.path.basename(path)
        filtered_dict["packagetype"] = meta_dict["filetype"]
        package = cls(**filtered_dict)
        package._checksum = package.checksum(path)
        package._checksum_type = CHECKSUM_TYPE
        return package
Exemplo n.º 3
0
def upload(dist, **kwargs):
    """Upload one or more files to the repository."""
    upload_settings = Settings(**kwargs)
    signatures = dict(
        (os.path.basename(d), d) for d in dist if d.endswith('.asc'))
    uploads = [d for d in dist if not d.endswith('.asc')]
    repository = upload_settings.create_repository()

    logger.info('Uploading distributions to {0}'.format(repository.get_url()))

    uploaded = 0
    for filename in uploads:
        package = PackageFile.from_filename(filename, '')
        skip_message = 'Skipping {0} because it appears to already exist'.format(
            package.basefilename)

        if upload_settings.skip_existing and repository.package_is_uploaded(
                package):
            logger.info(skip_message)
            continue

        signed_name = package.signed_basefilename
        if signed_name in signatures:
            package.add_gpg_signature(signatures[signed_name], signed_name)
        elif upload_settings.sign:
            package.sign(upload_settings.sign_with, upload_settings.identity)

        repository.upload(package)
        uploaded += 1

    if uploaded:
        repository.update_index()
Exemplo n.º 4
0
def register(package, repository, username, password, comment, config_file,
             cert, client_cert):
    config = utils.get_repository_from_config(config_file, repository)
    config["repository"] = utils.normalize_repository_url(
        config["repository"]
    )

    print("Registering package to {0}".format(config["repository"]))

    username = utils.get_username(username, config)
    password = utils.get_password(password, config)
    ca_cert = utils.get_cacert(cert, config)
    client_cert = utils.get_clientcert(client_cert, config)

    repository = Repository(config["repository"], username, password,
                            ca_cert, client_cert)

    if not os.path.exists(package):
        raise exc.PackageNotFound(
            '"{0}" does not exist on the file system.'.format(package)
        )

    resp = repository.register(PackageFile.from_filename(package, comment))
    repository.close()

    if resp.is_redirect:
        raise exc.RedirectDetected(
            ('"{0}" attempted to redirect to "{1}" during upload.'
             ' Aborting...').format(config["respository"],
                                    resp.headers["location"]))

    resp.raise_for_status()
Exemplo n.º 5
0
def upload(dists, repository, sign, identity, username, password, comment,
           sign_with, config_file, skip_existing, cert, client_cert):
    # Check that a nonsensical option wasn't given
    if not sign and identity:
        raise ValueError("sign must be given along with identity")

    dists = find_dists(dists)

    # Determine if the user has passed in pre-signed distributions
    signatures = dict(
        (os.path.basename(d), d) for d in dists if d.endswith(".asc"))
    uploads = [i for i in dists if not i.endswith(".asc")]

    config = utils.get_repository_from_config(config_file, repository)

    config["repository"] = utils.normalize_repository_url(config["repository"])

    print("Uploading distributions to {0}".format(config["repository"]))

    username = utils.get_username(username, config)
    password = utils.get_password(password, config)
    ca_cert = utils.get_cacert(cert, config)
    client_cert = utils.get_clientcert(client_cert, config)

    repository = Repository(config["repository"], username, password)
    repository.set_certificate_authority(ca_cert)
    repository.set_client_certificate(client_cert)

    for filename in uploads:
        package = PackageFile.from_filename(filename, comment)

        signed_name = package.signed_basefilename
        if signed_name in signatures:
            package.add_gpg_signature(signatures[signed_name], signed_name)
        elif sign:
            package.sign(sign_with, identity)

        resp = repository.upload(package)

        # Bug 92. If we get a redirect we should abort because something seems
        # funky. The behaviour is not well defined and redirects being issued
        # by PyPI should never happen in reality. This should catch malicious
        # redirects as well.
        if resp.is_redirect:
            raise exc.RedirectDetected(
                ('"{0}" attempted to redirect to "{1}" during upload.'
                 ' Aborting...').format(config["repository"],
                                        resp.headers["location"]))

        # Otherwise, raise an HTTPError based on the status code.
        if skip_upload(resp, skip_existing, package):
            print("  Skipping {0} because it appears to already exist".format(
                package.basefilename))
            continue

        resp.raise_for_status()

    # Bug 28. Try to silence a ResourceWarning by clearing the connection
    # pool.
    repository.close()
Exemplo n.º 6
0
def _check_file(filename, render_warning_stream):
    """Check given distribution."""
    warnings = []
    is_ok = True

    package = PackageFile.from_filename(filename, comment=None)

    metadata = package.metadata_dictionary()
    description = metadata["description"]
    description_content_type = metadata["description_content_type"]

    if description_content_type is None:
        warnings.append('`long_description_content_type` missing.  '
                        'defaulting to `text/x-rst`.')
        description_content_type = 'text/x-rst'

    content_type, params = cgi.parse_header(description_content_type)
    renderer = _RENDERERS.get(content_type, _RENDERERS[None])

    if description in {None, 'UNKNOWN\n\n\n'}:
        warnings.append('`long_description` missing.')
    elif renderer:
        rendering_result = renderer.render(description,
                                           stream=render_warning_stream,
                                           **params)
        if rendering_result is None:
            is_ok = False

    return warnings, is_ok
Exemplo n.º 7
0
    def _upload(self, package: PackageFile) -> Response:
        data = package.metadata_dictionary()
        data.update({
            # action
            ":action": "file_upload",
            "protocol_version": "1",
        })

        data_to_send = self._convert_data_to_list_of_tuples(data)

        print(f"Uploading {package.basefilename}")

        with open(package.filename, "rb") as fp:
            data_to_send.append((
                "content",
                (package.basefilename, fp, "application/octet-stream"),
            ))
            encoder = MultipartEncoder(data_to_send)
            with ProgressBar(total=encoder.len,
                             unit='B', unit_scale=True, unit_divisor=1024,
                             miniters=1, file=sys.stdout,
                             disable=self.disable_progress_bar) as bar:
                monitor = MultipartEncoderMonitor(
                    encoder, lambda monitor: bar.update_to(monitor.bytes_read)
                )

                resp = self.session.post(
                    self.url,
                    data=monitor,
                    allow_redirects=False,
                    headers={'Content-Type': monitor.content_type},
                )

        return resp
Exemplo n.º 8
0
def register(package, repository, username, password, comment, config_file,
             cert, client_cert):
    config = utils.get_repository_from_config(config_file, repository)
    config["repository"] = utils.normalize_repository_url(config["repository"])

    print("Registering package to {0}".format(config["repository"]))

    username = utils.get_username(username, config)
    password = utils.get_password(password, config)
    ca_cert = utils.get_cacert(cert, config)
    client_cert = utils.get_clientcert(client_cert, config)

    repository = Repository(config["repository"], username, password)
    repository.set_certificate_authority(ca_cert)
    repository.set_client_certificate(client_cert)

    if not os.path.exists(package):
        raise exc.PackageNotFound(
            '"{0}" does not exist on the file system.'.format(package))

    resp = repository.register(PackageFile.from_filename(package, comment))
    repository.close()

    if resp.is_redirect:
        raise exc.RedirectDetected(
            ('"{0}" attempted to redirect to "{1}" during registration.'
             ' Aborting...').format(config["repository"],
                                    resp.headers["location"]))

    resp.raise_for_status()
Exemplo n.º 9
0
def check(dists, output_stream=sys.stdout):
    uploads = [i for i in _find_dists(dists) if not i.endswith(".asc")]
    stream = _WarningStream()
    failure = False

    for filename in uploads:
        output_stream.write("Checking distribution %s: " % filename)
        package = PackageFile.from_filename(filename, comment=None)

        metadata = package.metadata_dictionary()
        content_type, parameters = cgi.parse_header(
            metadata.get("description_content_type") or "")

        # Get the appropriate renderer
        renderer = _RENDERERS.get(content_type, readme_renderer.txt)

        # Actually render the given value
        rendered = renderer.render(metadata.get("description"),
                                   stream=stream,
                                   **parameters)

        if rendered is None:
            failure = True
            output_stream.write("Failed\n")
            output_stream.write(
                "The project's long_description has invalid markup which will "
                "not be rendered on PyPI. The following syntax errors were "
                "detected:\n%s" % stream)
        else:
            output_stream.write("Passed\n")

    return failure
Exemplo n.º 10
0
def upload(dists, repository, sign, identity, username, password, comment,
           sign_with, config_file, skip_existing):
    # Check that a nonsensical option wasn't given
    if not sign and identity:
        raise ValueError("sign must be given along with identity")

    dists = find_dists(dists)

    # Determine if the user has passed in pre-signed distributions
    signatures = dict(
        (os.path.basename(d), d) for d in dists if d.endswith(".asc")
    )
    uploads = [i for i in dists if not i.endswith(".asc")]

    config = utils.get_repository_from_config(config_file, repository)

    config["repository"] = utils.normalize_repository_url(
        config["repository"]
    )

    print("Uploading distributions to {0}".format(config["repository"]))

    username = utils.get_username(username, config)
    password = utils.get_password(password, config)

    repository = Repository(config["repository"], username, password)

    for filename in uploads:
        package = PackageFile.from_filename(filename, comment)

        signed_name = package.signed_basefilename
        if signed_name in signatures:
            package.add_gpg_signature(signatures[signed_name], signed_name)
        elif sign:
            package.sign(sign_with, identity)

        resp = repository.upload(package)

        # Bug 92. If we get a redirect we should abort because something seems
        # funky. The behaviour is not well defined and redirects being issued
        # by PyPI should never happen in reality. This should catch malicious
        # redirects as well.
        if resp.is_redirect:
            raise exc.RedirectDetected(
                ('"{0}" attempted to redirect to "{1}" during upload.'
                 ' Aborting...').format(config["repository"],
                                        resp.headers["location"]))

        # Otherwise, raise an HTTPError based on the status code.
        if skip_upload(resp, skip_existing, package):
            print("  Skipping {0} because it appears to already exist".format(
                package.basefilename))
            continue

        resp.raise_for_status()

    # Bug 28. Try to silence a ResourceWarning by clearing the connection
    # pool.
    repository.close()
Exemplo n.º 11
0
def upload(upload_settings, dists):
    dists = _find_dists(dists)

    # Determine if the user has passed in pre-signed distributions
    signatures = dict(
        (os.path.basename(d), d) for d in dists if d.endswith(".asc")
    )
    uploads = [i for i in dists if not i.endswith(".asc")]
    upload_settings.check_repository_url()
    repository_url = upload_settings.repository_config['repository']

    print("Uploading distributions to {0}".format(repository_url))

    repository = upload_settings.create_repository()

    for filename in uploads:
        package = PackageFile.from_filename(filename, upload_settings.comment)
        skip_message = (
            "  Skipping {0} because it appears to already exist".format(
                package.basefilename)
        )

        # Note: The skip_existing check *needs* to be first, because otherwise
        #       we're going to generate extra HTTP requests against a hardcoded
        #       URL for no reason.
        if (upload_settings.skip_existing and
                repository.package_is_uploaded(package)):
            print(skip_message)
            continue

        signed_name = package.signed_basefilename
        if signed_name in signatures:
            package.add_gpg_signature(signatures[signed_name], signed_name)
        elif upload_settings.sign:
            package.sign(upload_settings.sign_with, upload_settings.identity)

        resp = repository.upload(package)

        # Bug 92. If we get a redirect we should abort because something seems
        # funky. The behaviour is not well defined and redirects being issued
        # by PyPI should never happen in reality. This should catch malicious
        # redirects as well.
        if resp.is_redirect:
            raise exc.RedirectDetected(
                ('"{0}" attempted to redirect to "{1}" during upload.'
                 ' Aborting...').format(repository_url,
                                        resp.headers["location"]))

        if skip_upload(resp, upload_settings.skip_existing, package):
            print(skip_message)
            continue
        utils.check_status_code(resp, upload_settings.verbose)

    # Bug 28. Try to silence a ResourceWarning by clearing the connection
    # pool.
    repository.close()

    return True
Exemplo n.º 12
0
 def _retry_twine(self, twine_command, server, filename):
     repository = self._get_repository(server)
     package_file = PackageFile.from_filename(filename, comment=None)
     if twine_command == 'register':
         # Register the package.
         twine_function = repository.register
         twine_args = (package_file, )
     elif twine_command == 'upload':
         try:
             already_uploaded = repository.package_is_uploaded(package_file)
         except ValueError:
             # For a new package, the call fails, at least with twine 1.8.1.
             # This is the same as when calling `twine --skip-existing` on
             # the command line.  See
             # https://github.com/pypa/twine/issues/220
             logger.warning('Error calling package_is_uploaded from twine. '
                            'Probably new project. Will try uploading.')
             already_uploaded = False
         if already_uploaded:
             logger.warning(
                 'A file %s has already been uploaded. Ignoring.', filename)
             return
         twine_function = repository.upload
         twine_args = (package_file, )
     else:
         print(Fore.RED + "Unknown twine command: %s" % twine_command)
         sys.exit(1)
     response = twine_function(*twine_args)
     if response is not None and response.status_code == codes.OK:
         return
     # if we get 409 that's conflict (which I'm going to interpret as the package has already been uploaded
     if response is not None and response.status_code == codes.CONFLICT:
         return
     # Something went wrong.  Close repository.
     repository.close()
     self._drop_repository(server)
     if response is not None:
         # Some errors reported by PyPI after register or upload may be
         # fine.  The register command is not really needed anymore with the
         # new PyPI.  See https://github.com/pypa/twine/issues/200
         # This might change, but for now the register command fails.
         if (twine_command == 'register'
                 and response.reason == 'This API is no longer supported, '
                 'instead simply upload the file.'):
             return
         # Show the error.
         print(Fore.RED + "Response status code: %s" % response.status_code)
         print(Fore.RED + "Reason: %s" % response.reason)
     print(Fore.RED + "There were errors or warnings.")
     logger.exception("Package %s has failed.", twine_command)
     retry = utils.retry_yes_no(['twine', twine_command])
     if retry:
         logger.info("Retrying.")
         # Reload the pypi config so changes that the user has made to
         # influence the retry can take effect.
         self.pypiconfig.reload()
         return self._retry_twine(twine_command, server, filename)
Exemplo n.º 13
0
Arquivo: upload.py Projeto: pypa/twine
def upload(upload_settings, dists):
    dists = _find_dists(dists)

    # Determine if the user has passed in pre-signed distributions
    signatures = {os.path.basename(d): d for d in dists if d.endswith(".asc")}
    uploads = [i for i in dists if not i.endswith(".asc")]
    upload_settings.check_repository_url()
    repository_url = upload_settings.repository_config['repository']

    print("Uploading distributions to {}".format(repository_url))

    repository = upload_settings.create_repository()

    for filename in uploads:
        package = PackageFile.from_filename(filename, upload_settings.comment)
        skip_message = (
            "  Skipping {} because it appears to already exist".format(
                package.basefilename)
        )

        # Note: The skip_existing check *needs* to be first, because otherwise
        #       we're going to generate extra HTTP requests against a hardcoded
        #       URL for no reason.
        if (upload_settings.skip_existing and
                repository.package_is_uploaded(package)):
            print(skip_message)
            continue

        signed_name = package.signed_basefilename
        if signed_name in signatures:
            package.add_gpg_signature(signatures[signed_name], signed_name)
        elif upload_settings.sign:
            package.sign(upload_settings.sign_with, upload_settings.identity)

        resp = repository.upload(package)

        # Bug 92. If we get a redirect we should abort because something seems
        # funky. The behaviour is not well defined and redirects being issued
        # by PyPI should never happen in reality. This should catch malicious
        # redirects as well.
        if resp.is_redirect:
            raise exceptions.RedirectDetected(
                ('"{0}" attempted to redirect to "{1}" during upload.'
                 ' Aborting...').format(repository_url,
                                        resp.headers["location"]))

        if skip_upload(resp, upload_settings.skip_existing, package):
            print(skip_message)
            continue
        utils.check_status_code(resp, upload_settings.verbose)

    # Bug 28. Try to silence a ResourceWarning by clearing the connection
    # pool.
    repository.close()
Exemplo n.º 14
0
 def _get_packages(self, safe_name):
     """Yield (PackageFile, metadata) for each package in the project"""
     prefix = '{0}{1}/'.format(self.prefix, safe_name)
     logger.info('Looking for packages in {}'.format(prefix))
     with tempfile.TemporaryDirectory() as temp_dir:
         for page in self.client.get_paginator('list_objects').paginate(
                 Bucket=self.bucket, Prefix=prefix, Delimiter='/'):
             for item in page.get('Contents', []):
                 key = item['Key'].replace(prefix, '', 1)
                 if not (key == '' or key.endswith('json')
                         or key.endswith('.asc')):
                     try:
                         filename = os.path.join(temp_dir, key)
                         logger.info('Downloading {0}'.format(item['Key']))
                         self.client.download_file(Bucket=self.bucket,
                                                   Key=item['Key'],
                                                   Filename=filename)
                         package = PackageFile.from_filename(filename, '')
                         try:
                             self.client.head_object(Bucket=self.bucket,
                                                     Key=item['Key'] +
                                                     '.asc')
                             logger.info(
                                 'Downloading {0}'.format(item['Key'] +
                                                          '.asc'))
                             self.client.download_file(
                                 Bucket=self.bucket,
                                 Key=item['Key'] + '.asc',
                                 Filename=filename + '.asc')
                             package.add_gpg_signature(
                                 package.signed_filename,
                                 package.signed_basefilename)
                         except ClientError as e:
                             if e.response['Error']['Code'] in [
                                     '403', '404'
                             ]:
                                 logger.debug(
                                     'No GPG signature for {0}'.format(
                                         item['Key']))
                             else:
                                 raise e
                         yield (package, item)
                     except InvalidDistribution as e:
                         logger.warn('Skipping {0}: {1}'.format(
                             item['Key'], e))
                     except ClientError:
                         logger.error('Failed to download {0}'.format(
                             item['Key']),
                                      exc_info=True)
                     finally:
                         if os.path.exists(filename):
                             os.unlink(filename)
Exemplo n.º 15
0
    def upload(self, poet):
        """
        Upload packages represented by a Poet instance.

        :param poet: The Poet instance representing the package.
        :type poet: poet.poet.Poet
        """
        skip_existing = False
        dists = find_dists([
            os.path.join(os.path.join(poet.base_dir, 'dist'),
                         '{}-{}*'.format(poet.name, poet.version))
        ])

        uploads = [i for i in dists if not i.endswith(".asc")]

        for filename in uploads:
            package = PackageFile.from_filename(filename, None)
            skip_message = (
                " - Skipping <comment>{0}</> because it appears to already exist"
                .format(package.basefilename))

            # Note: The skip_existing check *needs* to be first, because otherwise
            #       we're going to generate extra HTTP requests against a hardcoded
            #       URL for no reason.
            if skip_existing and self._repository.package_is_uploaded(package):
                self._output.writeln(skip_message)
                continue

            resp = self._repository.upload(package)

            # Bug 92. If we get a redirect we should abort because something seems
            # funky. The behaviour is not well defined and redirects being issued
            # by PyPI should never happen in reality. This should catch malicious
            # redirects as well.
            if resp.is_redirect:
                raise RedirectDetected(
                    ('"{0}" attempted to redirect to "{1}" during upload.'
                     ' Aborting...').format(self._repository.url,
                                            resp.headers["location"]))

            if skip_upload(resp, skip_existing, package):
                self._output.writeln(skip_message)

                continue

            twine.utils.check_status_code(resp)

        # Bug 28. Try to silence a ResourceWarning by clearing the connection
        # pool.
        self._repository.close()
Exemplo n.º 16
0
    def register(self, package: package_file.PackageFile) -> requests.Response:
        data = package.metadata_dictionary()
        data.update({":action": "submit", "protocol_version": "1"})

        print(f"Registering {package.basefilename}")

        data_to_send = self._convert_data_to_list_of_tuples(data)
        encoder = requests_toolbelt.MultipartEncoder(data_to_send)
        resp = self.session.post(
            self.url,
            data=encoder,
            allow_redirects=False,
            headers={"Content-Type": encoder.content_type},
        )
        # Bug 28. Try to silence a ResourceWarning by releasing the socket.
        resp.close()
        return resp
Exemplo n.º 17
0
 def _retry_twine(self, twine_command, server, filename):
     repository = self._get_repository(server)
     package_file = PackageFile.from_filename(filename, comment=None)
     if twine_command == 'register':
         # Register the package.
         twine_function = repository.register
         twine_args = (package_file, )
     elif twine_command == 'upload':
         # Note: we assume here that calling package_is_uploaded does not
         # give an error, and that there is no reason to retry it.
         if repository.package_is_uploaded(package_file):
             logger.warn('A file %s has already been uploaded. Ignoring.',
                         filename)
             return
         twine_function = repository.upload
         twine_args = (package_file, )
     else:
         print(Fore.RED + "Unknown twine command: %s" % twine_command)
         sys.exit(1)
     response = twine_function(*twine_args)
     if response is not None and response.status_code == codes.OK:
         return
     # Something went wrong.  Close repository.
     repository.close()
     self._drop_repository(server)
     if response is not None:
         # Some errors reported by PyPI after register or upload may be
         # fine.  The register command is not really needed anymore with the
         # new PyPI.  See https://github.com/pypa/twine/issues/200
         # This might change, but for now the register command fails.
         if (twine_command == 'register'
                 and response.reason == 'This API is no longer supported, '
                 'instead simply upload the file.'):
             return
         # Show the error.
         print(Fore.RED + "Response status code: %s" % response.status_code)
         print(Fore.RED + "Reason: %s" % response.reason)
     print(Fore.RED + "There were errors or warnings.")
     logger.exception("Package %s has failed.", twine_command)
     retry = utils.retry_yes_no('twine %s' % twine_command)
     if retry:
         logger.info("Retrying.")
         return self._retry_twine(twine_command, server, filename)
Exemplo n.º 18
0
 def _retry_twine(self, twine_command, server, filename):
     repository = self._get_repository(server)
     package_file = PackageFile.from_filename(filename, comment=None)
     if twine_command == "register":
         # Register the package.
         twine_function = repository.register
         twine_args = (package_file,)
     elif twine_command == "upload":
         # Note: we assume here that calling package_is_uploaded does not
         # give an error, and that there is no reason to retry it.
         if repository.package_is_uploaded(package_file):
             logger.warn("A file %s has already been uploaded. Ignoring.", filename)
             return
         twine_function = repository.upload
         twine_args = (package_file,)
     else:
         print(Fore.RED + "Unknown twine command: %s" % twine_command)
         sys.exit(1)
     response = twine_function(*twine_args)
     if response is not None and response.status_code == codes.OK:
         return
     # Something went wrong.  Close repository.
     repository.close()
     self._drop_repository(server)
     if response is not None:
         # Some errors reported by PyPI after register or upload may be
         # fine.  The register command is not really needed anymore with the
         # new PyPI.  See https://github.com/pypa/twine/issues/200
         # This might change, but for now the register command fails.
         if (
             twine_command == "register"
             and response.reason == "This API is no longer supported, " "instead simply upload the file."
         ):
             return
         # Show the error.
         print(Fore.RED + "Response status code: %s" % response.status_code)
         print(Fore.RED + "Reason: %s" % response.reason)
     print(Fore.RED + "There were errors or warnings.")
     logger.exception("Package %s has failed.", twine_command)
     retry = utils.retry_yes_no("twine %s" % twine_command)
     if retry:
         logger.info("Retrying.")
         return self._retry_twine(twine_command, server, filename)
Exemplo n.º 19
0
def check(dists, output_stream=sys.stdout):
    uploads = [i for i in _find_dists(dists) if not i.endswith(".asc")]
    stream = _WarningStream()
    failure = False

    for filename in uploads:
        output_stream.write("Checking distribution %s: " % filename)
        package = PackageFile.from_filename(filename, comment=None)

        metadata = package.metadata_dictionary()
        description = metadata["description"]
        description_content_type = metadata["description_content_type"]

        if description_content_type is None:
            output_stream.write(
                'warning: `long_description_content_type` missing.  '
                'defaulting to `text/x-rst`.\n'
            )
            description_content_type = 'text/x-rst'

        content_type, params = cgi.parse_header(description_content_type)
        renderer = _RENDERERS.get(content_type, _RENDERERS[None])

        if description in {None, 'UNKNOWN\n\n\n'}:
            output_stream.write('warning: `long_description` missing.\n')
            output_stream.write("Passed\n")
        else:
            if (
                renderer
                and renderer.render(description, stream=stream, **params)
                is None
            ):
                failure = True
                output_stream.write("Failed\n")
                output_stream.write(
                    "The project's long_description has invalid markup which "
                    "will not be rendered on PyPI. The following syntax "
                    "errors were detected:\n%s" % stream
                )
            else:
                output_stream.write("Passed\n")

    return failure
Exemplo n.º 20
0
Arquivo: check.py Projeto: pypa/twine
def check(dists, output_stream=sys.stdout):
    uploads = [i for i in _find_dists(dists) if not i.endswith(".asc")]
    stream = _WarningStream()
    failure = False

    for filename in uploads:
        output_stream.write("Checking distribution %s: " % filename)
        package = PackageFile.from_filename(filename, comment=None)

        metadata = package.metadata_dictionary()
        description = metadata["description"]
        description_content_type = metadata["description_content_type"]

        if description_content_type is None:
            output_stream.write(
                'warning: `long_description_content_type` missing.  '
                'defaulting to `text/x-rst`.\n'
            )
            description_content_type = 'text/x-rst'

        content_type, params = cgi.parse_header(description_content_type)
        renderer = _RENDERERS.get(content_type, _RENDERERS[None])

        if description in {None, 'UNKNOWN\n\n\n'}:
            output_stream.write('warning: `long_description` missing.\n')
            output_stream.write("Passed\n")
        else:
            if (
                renderer
                and renderer.render(description, stream=stream, **params)
                is None
            ):
                failure = True
                output_stream.write("Failed\n")
                output_stream.write(
                    "The project's long_description has invalid markup which "
                    "will not be rendered on PyPI. The following syntax "
                    "errors were detected:\n%s" % stream
                )
            else:
                output_stream.write("Passed\n")

    return failure
Exemplo n.º 21
0
def get_hashes(dist_path: str,
               simple_url: str) -> Tuple[str, Union[str, None]]:
    """Get local and remote hashes for a distribution

    Returns a pair (local_hash, remote_hash).

    The algorithm for the hashes returned is based on the information provided
    by remote's Simple API. Based on PEP 503, the API can return any type of a
    hash, as long as it is supported by the builtin hashlib module.

    If the Simple API doesn't return any hash, raises UnsupportedAPIError.

    If the package couldn't be found, an md5 has of local package is returned
    along with a None, i.e. ('<md5>', None).
    """
    client = PyPISimple(simple_url)
    package = PackageFile.from_filename(dist_path, comment=None)
    name = package.safe_name

    filename = os.path.basename(dist_path)
    _, local_version, local_package_type = parse_filename(filename,
                                                          project_hint=name)

    remote_dists = client.get_project_files(name)

    for remote_dist in remote_dists:
        if (versions_equal(local_version, remote_dist.version)
                and remote_dist.package_type == local_package_type):
            break
    else:
        return hash_file(dist_path, hashlib.md5), None
    try:
        algo, remote_hash = url_hash_fragment(remote_dist.url)
    except Exception as exc:
        raise UnsupportedAPIError("API doesn't support hashes.") from exc

    if algo not in hashlib.algorithms_guaranteed:
        raise UnsupportedAPIError(f"API returned an unsupported hash: {algo}.")

    hashlib_cls = getattr(hashlib, algo)
    local_hash = hash_file(dist_path, hashlib_cls)

    return local_hash, remote_hash
Exemplo n.º 22
0
def register(register_settings, package):
    repository_url = register_settings.repository_config['repository']

    print("Registering package to {0}".format(repository_url))
    repository = register_settings.create_repository()

    if not os.path.exists(package):
        raise exc.PackageNotFound(
            '"{0}" does not exist on the file system.'.format(package))

    resp = repository.register(
        PackageFile.from_filename(package, register_settings.comment))
    repository.close()

    if resp.is_redirect:
        raise exc.RedirectDetected(
            ('"{0}" attempted to redirect to "{1}" during registration.'
             ' Aborting...').format(repository_url, resp.headers["location"]))

    resp.raise_for_status()
Exemplo n.º 23
0
def register(register_settings, package):
    repository_url = register_settings.repository_config['repository']

    print(f"Registering package to {repository_url}")
    repository = register_settings.create_repository()

    if not os.path.exists(package):
        raise exceptions.PackageNotFound(
            f'"{package}" does not exist on the file system.')

    resp = repository.register(
        PackageFile.from_filename(package, register_settings.comment))
    repository.close()

    if resp.is_redirect:
        raise exceptions.RedirectDetected.from_args(
            repository_url,
            resp.headers["location"],
        )

    resp.raise_for_status()
Exemplo n.º 24
0
def register(register_settings, package):
    repository_url = register_settings.repository_config['repository']

    print("Registering package to {}".format(repository_url))
    repository = register_settings.create_repository()

    if not os.path.exists(package):
        raise exceptions.PackageNotFound(
            '"{}" does not exist on the file system.'.format(package)
        )

    resp = repository.register(
        PackageFile.from_filename(package, register_settings.comment)
    )
    repository.close()

    if resp.is_redirect:
        raise exceptions.RedirectDetected(
            ('"{0}" attempted to redirect to "{1}" during registration.'
             ' Aborting...').format(repository_url,
                                    resp.headers["location"]))

    resp.raise_for_status()
Exemplo n.º 25
0
    def register(self, poet):
        """
        Register a package represented by a Poet instance.
        
        :param poet: The Poet instance representing the package.
        :type poet: poet.poet.Poet
        """
        package = os.path.join(poet.base_dir, 'dist', poet.archive)

        if not os.path.exists(package):
            raise PackageNotFound(
                '"{0}" does not exist on the file system.'.format(package))

        resp = self._repository.register(
            PackageFile.from_filename(package, None))
        self._repository.close()

        if resp.is_redirect:
            raise RedirectDetected(
                ('"{0}" attempted to redirect to "{1}" during registration.'
                 ' Aborting...').format(self._repository.url,
                                        resp.headers["location"]))

        resp.raise_for_status()
Exemplo n.º 26
0
def upload(dists, repository, sign, identity, username, password, comment,
           sign_with, config_file, skip_existing, cert, client_cert,
           repository_url):
    # Check that a nonsensical option wasn't given
    if not sign and identity:
        raise ValueError("sign must be given along with identity")

    dists = find_dists(dists)

    # Determine if the user has passed in pre-signed distributions
    signatures = dict(
        (os.path.basename(d), d) for d in dists if d.endswith(".asc")
    )
    uploads = [i for i in dists if not i.endswith(".asc")]

    config = utils.get_repository_from_config(
        config_file,
        repository,
        repository_url,
    )

    config["repository"] = utils.normalize_repository_url(
        config["repository"]
    )

    print("Uploading distributions to {0}".format(config["repository"]))

    username = utils.get_username(username, config)
    password = utils.get_password(
        config["repository"], username, password, config,
    )
    ca_cert = utils.get_cacert(cert, config)
    client_cert = utils.get_clientcert(client_cert, config)

    repository = Repository(config["repository"], username, password)
    repository.set_certificate_authority(ca_cert)
    repository.set_client_certificate(client_cert)

    for filename in uploads:
        package = PackageFile.from_filename(filename, comment)
        skip_message = (
            "  Skipping {0} because it appears to already exist".format(
                package.basefilename)
        )

        # Note: The skip_existing check *needs* to be first, because otherwise
        #       we're going to generate extra HTTP requests against a hardcoded
        #       URL for no reason.
        if skip_existing and repository.package_is_uploaded(package):
            print(skip_message)
            continue

        signed_name = package.signed_basefilename
        if signed_name in signatures:
            package.add_gpg_signature(signatures[signed_name], signed_name)
        elif sign:
            package.sign(sign_with, identity)

        resp = repository.upload(package)

        # Bug 92. If we get a redirect we should abort because something seems
        # funky. The behaviour is not well defined and redirects being issued
        # by PyPI should never happen in reality. This should catch malicious
        # redirects as well.
        if resp.is_redirect:
            raise exc.RedirectDetected(
                ('"{0}" attempted to redirect to "{1}" during upload.'
                 ' Aborting...').format(config["repository"],
                                        resp.headers["location"]))

        if skip_upload(resp, skip_existing, package):
            print(skip_message)
            continue

        utils.check_status_code(resp)

    # Bug 28. Try to silence a ResourceWarning by clearing the connection
    # pool.
    repository.close()
Exemplo n.º 27
0
def upload(upload_settings, dists):
    dists = _find_dists(dists)

    # Determine if the user has passed in pre-signed distributions
    signatures = {os.path.basename(d): d for d in dists if d.endswith(".asc")}
    uploads = [i for i in dists if not i.endswith(".asc")]
    upload_settings.check_repository_url()
    repository_url = upload_settings.repository_config['repository']

    print(f"Uploading distributions to {repository_url}")

    repository = upload_settings.create_repository()
    uploaded_packages = []

    for filename in uploads:
        package = PackageFile.from_filename(filename, upload_settings.comment)
        skip_message = (
            "  Skipping {} because it appears to already exist".format(
                package.basefilename))

        # Note: The skip_existing check *needs* to be first, because otherwise
        #       we're going to generate extra HTTP requests against a hardcoded
        #       URL for no reason.
        if (upload_settings.skip_existing
                and repository.package_is_uploaded(package)):
            print(skip_message)
            continue

        signed_name = package.signed_basefilename
        if signed_name in signatures:
            package.add_gpg_signature(signatures[signed_name], signed_name)
        elif upload_settings.sign:
            package.sign(upload_settings.sign_with, upload_settings.identity)

        # Suppress TLS verification warning on trusted custom certs
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            resp = repository.upload(package)

        # Bug 92. If we get a redirect we should abort because something seems
        # funky. The behaviour is not well defined and redirects being issued
        # by PyPI should never happen in reality. This should catch malicious
        # redirects as well.
        if resp.is_redirect:
            raise exceptions.RedirectDetected.from_args(
                repository_url,
                resp.headers["location"],
            )

        if skip_upload(resp, upload_settings.skip_existing, package):
            print(skip_message)
            continue

        utils.check_status_code(resp, upload_settings.verbose)

        uploaded_packages.append(package)

    release_urls = repository.release_urls(uploaded_packages)
    if release_urls:
        print('\nView at:')
        for url in release_urls:
            print(url)

    # Bug 28. Try to silence a ResourceWarning by clearing the connection
    # pool.
    repository.close()
Exemplo n.º 28
0
def upload(dists, repository, sign, identity, username, password, comment,
           sign_with, config_file, skip_existing, cert, client_cert,
           repository_url):
    # Check that a nonsensical option wasn't given
    if not sign and identity:
        raise ValueError("sign must be given along with identity")

    dists = find_dists(dists)

    # Determine if the user has passed in pre-signed distributions
    signatures = dict(
        (os.path.basename(d), d) for d in dists if d.endswith(".asc")
    )
    uploads = [i for i in dists if not i.endswith(".asc")]

    config = utils.get_repository_from_config(
        config_file,
        repository,
        repository_url,
    )

    config["repository"] = utils.normalize_repository_url(
        config["repository"]
    )

    print("Uploading distributions to {0}".format(config["repository"]))

    if config["repository"].startswith((LEGACY_PYPI, LEGACY_TEST_PYPI)):
        raise exc.UploadToDeprecatedPyPIDetected(
            "You're trying to upload to the legacy PyPI site '{0}'. "
            "Uploading to those sites is deprecated. \n "
            "The new sites are pypi.org and test.pypi.org. Try using "
            "{1} (or {2}) to upload your packages instead. "
            "These are the default URLs for Twine now. \n More at "
            "https://packaging.python.org/guides/migrating-to-pypi-org/ "
            ".".format(
                config["repository"],
                utils.DEFAULT_REPOSITORY,
                utils.TEST_REPOSITORY
                )
            )

    username = utils.get_username(username, config)
    password = utils.get_password(
        config["repository"], username, password, config,
    )
    ca_cert = utils.get_cacert(cert, config)
    client_cert = utils.get_clientcert(client_cert, config)

    repository = Repository(config["repository"], username, password)
    repository.set_certificate_authority(ca_cert)
    repository.set_client_certificate(client_cert)

    for filename in uploads:
        package = PackageFile.from_filename(filename, comment)
        skip_message = (
            "  Skipping {0} because it appears to already exist".format(
                package.basefilename)
        )

        # Note: The skip_existing check *needs* to be first, because otherwise
        #       we're going to generate extra HTTP requests against a hardcoded
        #       URL for no reason.
        if skip_existing and repository.package_is_uploaded(package):
            print(skip_message)
            continue

        signed_name = package.signed_basefilename
        if signed_name in signatures:
            package.add_gpg_signature(signatures[signed_name], signed_name)
        elif sign:
            package.sign(sign_with, identity)

        resp = repository.upload(package)

        # Bug 92. If we get a redirect we should abort because something seems
        # funky. The behaviour is not well defined and redirects being issued
        # by PyPI should never happen in reality. This should catch malicious
        # redirects as well.
        if resp.is_redirect:
            raise exc.RedirectDetected(
                ('"{0}" attempted to redirect to "{1}" during upload.'
                 ' Aborting...').format(config["repository"],
                                        resp.headers["location"]))

        if skip_upload(resp, skip_existing, package):
            print(skip_message)
            continue

        utils.check_status_code(resp)

    # Bug 28. Try to silence a ResourceWarning by clearing the connection
    # pool.
    repository.close()