def from_archive(cls, path):
"""
Create and return (but do not save) an instance of a Python package object from the package
itself.
Twine is smart enough to crack open a tarball, zip, or wheel and parse the metadata that is
contained in the package.
:param path: path to the package
:type path: basestring
:return: instance of a package
:rtype: pulp_python.plugins.models.Package
"""
meta_dict = PackageFile.from_filename(
path, comment='').metadata_dictionary()
filtered_dict = {}
# Use only a subset of the attributes in the metadata to keep package minimal.
for key, value in meta_dict.iteritems():
if key in PACKAGE_ATTRS:
filtered_dict[key] = value
filtered_dict['filename'] = os.path.basename(path)
filtered_dict['packagetype'] = meta_dict['filetype']
package = cls(**filtered_dict)
package._checksum = package.checksum(path)
package._checksum_type = CHECKSUM_TYPE
return package
def from_archive(cls, path):
"""
Create and return (but do not save) an instance of a Python package object from the package
itself.
Twine is smart enough to crack open a tarball, zip, or wheel and parse the metadata that is
contained in the package.
:param path: path to the package
:type path: basestring
:return: instance of a package
:rtype: pulp_python.plugins.models.Package
"""
meta_dict = PackageFile.from_filename(path, comment="").metadata_dictionary()
filtered_dict = {}
# Use only a subset of the attributes in the metadata to keep package minimal.
for key, value in meta_dict.iteritems():
if key in PACKAGE_ATTRS:
filtered_dict[key] = value
filtered_dict["filename"] = os.path.basename(path)
filtered_dict["packagetype"] = meta_dict["filetype"]
package = cls(**filtered_dict)
package._checksum = package.checksum(path)
package._checksum_type = CHECKSUM_TYPE
return package
def upload(dist, **kwargs):
"""Upload one or more files to the repository."""
upload_settings = Settings(**kwargs)
signatures = dict(
(os.path.basename(d), d) for d in dist if d.endswith('.asc'))
uploads = [d for d in dist if not d.endswith('.asc')]
repository = upload_settings.create_repository()
logger.info('Uploading distributions to {0}'.format(repository.get_url()))
uploaded = 0
for filename in uploads:
package = PackageFile.from_filename(filename, '')
skip_message = 'Skipping {0} because it appears to already exist'.format(
package.basefilename)
if upload_settings.skip_existing and repository.package_is_uploaded(
package):
logger.info(skip_message)
continue
signed_name = package.signed_basefilename
if signed_name in signatures:
package.add_gpg_signature(signatures[signed_name], signed_name)
elif upload_settings.sign:
package.sign(upload_settings.sign_with, upload_settings.identity)
repository.upload(package)
uploaded += 1
if uploaded:
repository.update_index()
def register(package, repository, username, password, comment, config_file,
cert, client_cert):
config = utils.get_repository_from_config(config_file, repository)
config["repository"] = utils.normalize_repository_url(
config["repository"]
)
print("Registering package to {0}".format(config["repository"]))
username = utils.get_username(username, config)
password = utils.get_password(password, config)
ca_cert = utils.get_cacert(cert, config)
client_cert = utils.get_clientcert(client_cert, config)
repository = Repository(config["repository"], username, password,
ca_cert, client_cert)
if not os.path.exists(package):
raise exc.PackageNotFound(
'"{0}" does not exist on the file system.'.format(package)
)
resp = repository.register(PackageFile.from_filename(package, comment))
repository.close()
if resp.is_redirect:
raise exc.RedirectDetected(
('"{0}" attempted to redirect to "{1}" during upload.'
' Aborting...').format(config["respository"],
resp.headers["location"]))
resp.raise_for_status()
def upload(dists, repository, sign, identity, username, password, comment,
sign_with, config_file, skip_existing, cert, client_cert):
# Check that a nonsensical option wasn't given
if not sign and identity:
raise ValueError("sign must be given along with identity")
dists = find_dists(dists)
# Determine if the user has passed in pre-signed distributions
signatures = dict(
(os.path.basename(d), d) for d in dists if d.endswith(".asc"))
uploads = [i for i in dists if not i.endswith(".asc")]
config = utils.get_repository_from_config(config_file, repository)
config["repository"] = utils.normalize_repository_url(config["repository"])
print("Uploading distributions to {0}".format(config["repository"]))
username = utils.get_username(username, config)
password = utils.get_password(password, config)
ca_cert = utils.get_cacert(cert, config)
client_cert = utils.get_clientcert(client_cert, config)
repository = Repository(config["repository"], username, password)
repository.set_certificate_authority(ca_cert)
repository.set_client_certificate(client_cert)
for filename in uploads:
package = PackageFile.from_filename(filename, comment)
signed_name = package.signed_basefilename
if signed_name in signatures:
package.add_gpg_signature(signatures[signed_name], signed_name)
elif sign:
package.sign(sign_with, identity)
resp = repository.upload(package)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise exc.RedirectDetected(
('"{0}" attempted to redirect to "{1}" during upload.'
' Aborting...').format(config["repository"],
resp.headers["location"]))
# Otherwise, raise an HTTPError based on the status code.
if skip_upload(resp, skip_existing, package):
print(" Skipping {0} because it appears to already exist".format(
package.basefilename))
continue
resp.raise_for_status()
# Bug 28. Try to silence a ResourceWarning by clearing the connection
# pool.
repository.close()
def _check_file(filename, render_warning_stream):
"""Check given distribution."""
warnings = []
is_ok = True
package = PackageFile.from_filename(filename, comment=None)
metadata = package.metadata_dictionary()
description = metadata["description"]
description_content_type = metadata["description_content_type"]
if description_content_type is None:
warnings.append('`long_description_content_type` missing. '
'defaulting to `text/x-rst`.')
description_content_type = 'text/x-rst'
content_type, params = cgi.parse_header(description_content_type)
renderer = _RENDERERS.get(content_type, _RENDERERS[None])
if description in {None, 'UNKNOWN\n\n\n'}:
warnings.append('`long_description` missing.')
elif renderer:
rendering_result = renderer.render(description,
stream=render_warning_stream,
**params)
if rendering_result is None:
is_ok = False
return warnings, is_ok
def _upload(self, package: PackageFile) -> Response:
data = package.metadata_dictionary()
data.update({
# action
":action": "file_upload",
"protocol_version": "1",
})
data_to_send = self._convert_data_to_list_of_tuples(data)
print(f"Uploading {package.basefilename}")
with open(package.filename, "rb") as fp:
data_to_send.append((
"content",
(package.basefilename, fp, "application/octet-stream"),
))
encoder = MultipartEncoder(data_to_send)
with ProgressBar(total=encoder.len,
unit='B', unit_scale=True, unit_divisor=1024,
miniters=1, file=sys.stdout,
disable=self.disable_progress_bar) as bar:
monitor = MultipartEncoderMonitor(
encoder, lambda monitor: bar.update_to(monitor.bytes_read)
)
resp = self.session.post(
self.url,
data=monitor,
allow_redirects=False,
headers={'Content-Type': monitor.content_type},
)
return resp
def register(package, repository, username, password, comment, config_file,
cert, client_cert):
config = utils.get_repository_from_config(config_file, repository)
config["repository"] = utils.normalize_repository_url(config["repository"])
print("Registering package to {0}".format(config["repository"]))
username = utils.get_username(username, config)
password = utils.get_password(password, config)
ca_cert = utils.get_cacert(cert, config)
client_cert = utils.get_clientcert(client_cert, config)
repository = Repository(config["repository"], username, password)
repository.set_certificate_authority(ca_cert)
repository.set_client_certificate(client_cert)
if not os.path.exists(package):
raise exc.PackageNotFound(
'"{0}" does not exist on the file system.'.format(package))
resp = repository.register(PackageFile.from_filename(package, comment))
repository.close()
if resp.is_redirect:
raise exc.RedirectDetected(
('"{0}" attempted to redirect to "{1}" during registration.'
' Aborting...').format(config["repository"],
resp.headers["location"]))
resp.raise_for_status()
def check(dists, output_stream=sys.stdout):
uploads = [i for i in _find_dists(dists) if not i.endswith(".asc")]
stream = _WarningStream()
failure = False
for filename in uploads:
output_stream.write("Checking distribution %s: " % filename)
package = PackageFile.from_filename(filename, comment=None)
metadata = package.metadata_dictionary()
content_type, parameters = cgi.parse_header(
metadata.get("description_content_type") or "")
# Get the appropriate renderer
renderer = _RENDERERS.get(content_type, readme_renderer.txt)
# Actually render the given value
rendered = renderer.render(metadata.get("description"),
stream=stream,
**parameters)
if rendered is None:
failure = True
output_stream.write("Failed\n")
output_stream.write(
"The project's long_description has invalid markup which will "
"not be rendered on PyPI. The following syntax errors were "
"detected:\n%s" % stream)
else:
output_stream.write("Passed\n")
return failure
def upload(dists, repository, sign, identity, username, password, comment,
sign_with, config_file, skip_existing):
# Check that a nonsensical option wasn't given
if not sign and identity:
raise ValueError("sign must be given along with identity")
dists = find_dists(dists)
# Determine if the user has passed in pre-signed distributions
signatures = dict(
(os.path.basename(d), d) for d in dists if d.endswith(".asc")
)
uploads = [i for i in dists if not i.endswith(".asc")]
config = utils.get_repository_from_config(config_file, repository)
config["repository"] = utils.normalize_repository_url(
config["repository"]
)
print("Uploading distributions to {0}".format(config["repository"]))
username = utils.get_username(username, config)
password = utils.get_password(password, config)
repository = Repository(config["repository"], username, password)
for filename in uploads:
package = PackageFile.from_filename(filename, comment)
signed_name = package.signed_basefilename
if signed_name in signatures:
package.add_gpg_signature(signatures[signed_name], signed_name)
elif sign:
package.sign(sign_with, identity)
resp = repository.upload(package)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise exc.RedirectDetected(
('"{0}" attempted to redirect to "{1}" during upload.'
' Aborting...').format(config["repository"],
resp.headers["location"]))
# Otherwise, raise an HTTPError based on the status code.
if skip_upload(resp, skip_existing, package):
print(" Skipping {0} because it appears to already exist".format(
package.basefilename))
continue
resp.raise_for_status()
# Bug 28. Try to silence a ResourceWarning by clearing the connection
# pool.
repository.close()
def upload(upload_settings, dists):
dists = _find_dists(dists)
# Determine if the user has passed in pre-signed distributions
signatures = dict(
(os.path.basename(d), d) for d in dists if d.endswith(".asc")
)
uploads = [i for i in dists if not i.endswith(".asc")]
upload_settings.check_repository_url()
repository_url = upload_settings.repository_config['repository']
print("Uploading distributions to {0}".format(repository_url))
repository = upload_settings.create_repository()
for filename in uploads:
package = PackageFile.from_filename(filename, upload_settings.comment)
skip_message = (
" Skipping {0} because it appears to already exist".format(
package.basefilename)
)
# Note: The skip_existing check *needs* to be first, because otherwise
# we're going to generate extra HTTP requests against a hardcoded
# URL for no reason.
if (upload_settings.skip_existing and
repository.package_is_uploaded(package)):
print(skip_message)
continue
signed_name = package.signed_basefilename
if signed_name in signatures:
package.add_gpg_signature(signatures[signed_name], signed_name)
elif upload_settings.sign:
package.sign(upload_settings.sign_with, upload_settings.identity)
resp = repository.upload(package)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise exc.RedirectDetected(
('"{0}" attempted to redirect to "{1}" during upload.'
' Aborting...').format(repository_url,
resp.headers["location"]))
if skip_upload(resp, upload_settings.skip_existing, package):
print(skip_message)
continue
utils.check_status_code(resp, upload_settings.verbose)
# Bug 28. Try to silence a ResourceWarning by clearing the connection
# pool.
repository.close()
return True
def _retry_twine(self, twine_command, server, filename):
repository = self._get_repository(server)
package_file = PackageFile.from_filename(filename, comment=None)
if twine_command == 'register':
# Register the package.
twine_function = repository.register
twine_args = (package_file, )
elif twine_command == 'upload':
try:
already_uploaded = repository.package_is_uploaded(package_file)
except ValueError:
# For a new package, the call fails, at least with twine 1.8.1.
# This is the same as when calling `twine --skip-existing` on
# the command line. See
# https://github.com/pypa/twine/issues/220
logger.warning('Error calling package_is_uploaded from twine. '
'Probably new project. Will try uploading.')
already_uploaded = False
if already_uploaded:
logger.warning(
'A file %s has already been uploaded. Ignoring.', filename)
return
twine_function = repository.upload
twine_args = (package_file, )
else:
print(Fore.RED + "Unknown twine command: %s" % twine_command)
sys.exit(1)
response = twine_function(*twine_args)
if response is not None and response.status_code == codes.OK:
return
# if we get 409 that's conflict (which I'm going to interpret as the package has already been uploaded
if response is not None and response.status_code == codes.CONFLICT:
return
# Something went wrong. Close repository.
repository.close()
self._drop_repository(server)
if response is not None:
# Some errors reported by PyPI after register or upload may be
# fine. The register command is not really needed anymore with the
# new PyPI. See https://github.com/pypa/twine/issues/200
# This might change, but for now the register command fails.
if (twine_command == 'register'
and response.reason == 'This API is no longer supported, '
'instead simply upload the file.'):
return
# Show the error.
print(Fore.RED + "Response status code: %s" % response.status_code)
print(Fore.RED + "Reason: %s" % response.reason)
print(Fore.RED + "There were errors or warnings.")
logger.exception("Package %s has failed.", twine_command)
retry = utils.retry_yes_no(['twine', twine_command])
if retry:
logger.info("Retrying.")
# Reload the pypi config so changes that the user has made to
# influence the retry can take effect.
self.pypiconfig.reload()
return self._retry_twine(twine_command, server, filename)
def upload(upload_settings, dists):
dists = _find_dists(dists)
# Determine if the user has passed in pre-signed distributions
signatures = {os.path.basename(d): d for d in dists if d.endswith(".asc")}
uploads = [i for i in dists if not i.endswith(".asc")]
upload_settings.check_repository_url()
repository_url = upload_settings.repository_config['repository']
print("Uploading distributions to {}".format(repository_url))
repository = upload_settings.create_repository()
for filename in uploads:
package = PackageFile.from_filename(filename, upload_settings.comment)
skip_message = (
" Skipping {} because it appears to already exist".format(
package.basefilename)
)
# Note: The skip_existing check *needs* to be first, because otherwise
# we're going to generate extra HTTP requests against a hardcoded
# URL for no reason.
if (upload_settings.skip_existing and
repository.package_is_uploaded(package)):
print(skip_message)
continue
signed_name = package.signed_basefilename
if signed_name in signatures:
package.add_gpg_signature(signatures[signed_name], signed_name)
elif upload_settings.sign:
package.sign(upload_settings.sign_with, upload_settings.identity)
resp = repository.upload(package)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise exceptions.RedirectDetected(
('"{0}" attempted to redirect to "{1}" during upload.'
' Aborting...').format(repository_url,
resp.headers["location"]))
if skip_upload(resp, upload_settings.skip_existing, package):
print(skip_message)
continue
utils.check_status_code(resp, upload_settings.verbose)
# Bug 28. Try to silence a ResourceWarning by clearing the connection
# pool.
repository.close()
def _get_packages(self, safe_name):
"""Yield (PackageFile, metadata) for each package in the project"""
prefix = '{0}{1}/'.format(self.prefix, safe_name)
logger.info('Looking for packages in {}'.format(prefix))
with tempfile.TemporaryDirectory() as temp_dir:
for page in self.client.get_paginator('list_objects').paginate(
Bucket=self.bucket, Prefix=prefix, Delimiter='/'):
for item in page.get('Contents', []):
key = item['Key'].replace(prefix, '', 1)
if not (key == '' or key.endswith('json')
or key.endswith('.asc')):
try:
filename = os.path.join(temp_dir, key)
logger.info('Downloading {0}'.format(item['Key']))
self.client.download_file(Bucket=self.bucket,
Key=item['Key'],
Filename=filename)
package = PackageFile.from_filename(filename, '')
try:
self.client.head_object(Bucket=self.bucket,
Key=item['Key'] +
'.asc')
logger.info(
'Downloading {0}'.format(item['Key'] +
'.asc'))
self.client.download_file(
Bucket=self.bucket,
Key=item['Key'] + '.asc',
Filename=filename + '.asc')
package.add_gpg_signature(
package.signed_filename,
package.signed_basefilename)
except ClientError as e:
if e.response['Error']['Code'] in [
'403', '404'
]:
logger.debug(
'No GPG signature for {0}'.format(
item['Key']))
else:
raise e
yield (package, item)
except InvalidDistribution as e:
logger.warn('Skipping {0}: {1}'.format(
item['Key'], e))
except ClientError:
logger.error('Failed to download {0}'.format(
item['Key']),
exc_info=True)
finally:
if os.path.exists(filename):
os.unlink(filename)
def upload(self, poet):
"""
Upload packages represented by a Poet instance.
:param poet: The Poet instance representing the package.
:type poet: poet.poet.Poet
"""
skip_existing = False
dists = find_dists([
os.path.join(os.path.join(poet.base_dir, 'dist'),
'{}-{}*'.format(poet.name, poet.version))
])
uploads = [i for i in dists if not i.endswith(".asc")]
for filename in uploads:
package = PackageFile.from_filename(filename, None)
skip_message = (
" - Skipping <comment>{0}</> because it appears to already exist"
.format(package.basefilename))
# Note: The skip_existing check *needs* to be first, because otherwise
# we're going to generate extra HTTP requests against a hardcoded
# URL for no reason.
if skip_existing and self._repository.package_is_uploaded(package):
self._output.writeln(skip_message)
continue
resp = self._repository.upload(package)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise RedirectDetected(
('"{0}" attempted to redirect to "{1}" during upload.'
' Aborting...').format(self._repository.url,
resp.headers["location"]))
if skip_upload(resp, skip_existing, package):
self._output.writeln(skip_message)
continue
twine.utils.check_status_code(resp)
# Bug 28. Try to silence a ResourceWarning by clearing the connection
# pool.
self._repository.close()
def register(self, package: package_file.PackageFile) -> requests.Response:
data = package.metadata_dictionary()
data.update({":action": "submit", "protocol_version": "1"})
print(f"Registering {package.basefilename}")
data_to_send = self._convert_data_to_list_of_tuples(data)
encoder = requests_toolbelt.MultipartEncoder(data_to_send)
resp = self.session.post(
self.url,
data=encoder,
allow_redirects=False,
headers={"Content-Type": encoder.content_type},
)
# Bug 28. Try to silence a ResourceWarning by releasing the socket.
resp.close()
return resp
def _retry_twine(self, twine_command, server, filename):
repository = self._get_repository(server)
package_file = PackageFile.from_filename(filename, comment=None)
if twine_command == 'register':
# Register the package.
twine_function = repository.register
twine_args = (package_file, )
elif twine_command == 'upload':
# Note: we assume here that calling package_is_uploaded does not
# give an error, and that there is no reason to retry it.
if repository.package_is_uploaded(package_file):
logger.warn('A file %s has already been uploaded. Ignoring.',
filename)
return
twine_function = repository.upload
twine_args = (package_file, )
else:
print(Fore.RED + "Unknown twine command: %s" % twine_command)
sys.exit(1)
response = twine_function(*twine_args)
if response is not None and response.status_code == codes.OK:
return
# Something went wrong. Close repository.
repository.close()
self._drop_repository(server)
if response is not None:
# Some errors reported by PyPI after register or upload may be
# fine. The register command is not really needed anymore with the
# new PyPI. See https://github.com/pypa/twine/issues/200
# This might change, but for now the register command fails.
if (twine_command == 'register'
and response.reason == 'This API is no longer supported, '
'instead simply upload the file.'):
return
# Show the error.
print(Fore.RED + "Response status code: %s" % response.status_code)
print(Fore.RED + "Reason: %s" % response.reason)
print(Fore.RED + "There were errors or warnings.")
logger.exception("Package %s has failed.", twine_command)
retry = utils.retry_yes_no('twine %s' % twine_command)
if retry:
logger.info("Retrying.")
return self._retry_twine(twine_command, server, filename)
def _retry_twine(self, twine_command, server, filename):
repository = self._get_repository(server)
package_file = PackageFile.from_filename(filename, comment=None)
if twine_command == "register":
# Register the package.
twine_function = repository.register
twine_args = (package_file,)
elif twine_command == "upload":
# Note: we assume here that calling package_is_uploaded does not
# give an error, and that there is no reason to retry it.
if repository.package_is_uploaded(package_file):
logger.warn("A file %s has already been uploaded. Ignoring.", filename)
return
twine_function = repository.upload
twine_args = (package_file,)
else:
print(Fore.RED + "Unknown twine command: %s" % twine_command)
sys.exit(1)
response = twine_function(*twine_args)
if response is not None and response.status_code == codes.OK:
return
# Something went wrong. Close repository.
repository.close()
self._drop_repository(server)
if response is not None:
# Some errors reported by PyPI after register or upload may be
# fine. The register command is not really needed anymore with the
# new PyPI. See https://github.com/pypa/twine/issues/200
# This might change, but for now the register command fails.
if (
twine_command == "register"
and response.reason == "This API is no longer supported, " "instead simply upload the file."
):
return
# Show the error.
print(Fore.RED + "Response status code: %s" % response.status_code)
print(Fore.RED + "Reason: %s" % response.reason)
print(Fore.RED + "There were errors or warnings.")
logger.exception("Package %s has failed.", twine_command)
retry = utils.retry_yes_no("twine %s" % twine_command)
if retry:
logger.info("Retrying.")
return self._retry_twine(twine_command, server, filename)
def check(dists, output_stream=sys.stdout):
uploads = [i for i in _find_dists(dists) if not i.endswith(".asc")]
stream = _WarningStream()
failure = False
for filename in uploads:
output_stream.write("Checking distribution %s: " % filename)
package = PackageFile.from_filename(filename, comment=None)
metadata = package.metadata_dictionary()
description = metadata["description"]
description_content_type = metadata["description_content_type"]
if description_content_type is None:
output_stream.write(
'warning: `long_description_content_type` missing. '
'defaulting to `text/x-rst`.\n'
)
description_content_type = 'text/x-rst'
content_type, params = cgi.parse_header(description_content_type)
renderer = _RENDERERS.get(content_type, _RENDERERS[None])
if description in {None, 'UNKNOWN\n\n\n'}:
output_stream.write('warning: `long_description` missing.\n')
output_stream.write("Passed\n")
else:
if (
renderer
and renderer.render(description, stream=stream, **params)
is None
):
failure = True
output_stream.write("Failed\n")
output_stream.write(
"The project's long_description has invalid markup which "
"will not be rendered on PyPI. The following syntax "
"errors were detected:\n%s" % stream
)
else:
output_stream.write("Passed\n")
return failure
def check(dists, output_stream=sys.stdout):
uploads = [i for i in _find_dists(dists) if not i.endswith(".asc")]
stream = _WarningStream()
failure = False
for filename in uploads:
output_stream.write("Checking distribution %s: " % filename)
package = PackageFile.from_filename(filename, comment=None)
metadata = package.metadata_dictionary()
description = metadata["description"]
description_content_type = metadata["description_content_type"]
if description_content_type is None:
output_stream.write(
'warning: `long_description_content_type` missing. '
'defaulting to `text/x-rst`.\n'
)
description_content_type = 'text/x-rst'
content_type, params = cgi.parse_header(description_content_type)
renderer = _RENDERERS.get(content_type, _RENDERERS[None])
if description in {None, 'UNKNOWN\n\n\n'}:
output_stream.write('warning: `long_description` missing.\n')
output_stream.write("Passed\n")
else:
if (
renderer
and renderer.render(description, stream=stream, **params)
is None
):
failure = True
output_stream.write("Failed\n")
output_stream.write(
"The project's long_description has invalid markup which "
"will not be rendered on PyPI. The following syntax "
"errors were detected:\n%s" % stream
)
else:
output_stream.write("Passed\n")
return failure
def get_hashes(dist_path: str,
simple_url: str) -> Tuple[str, Union[str, None]]:
"""Get local and remote hashes for a distribution
Returns a pair (local_hash, remote_hash).
The algorithm for the hashes returned is based on the information provided
by remote's Simple API. Based on PEP 503, the API can return any type of a
hash, as long as it is supported by the builtin hashlib module.
If the Simple API doesn't return any hash, raises UnsupportedAPIError.
If the package couldn't be found, an md5 has of local package is returned
along with a None, i.e. ('<md5>', None).
"""
client = PyPISimple(simple_url)
package = PackageFile.from_filename(dist_path, comment=None)
name = package.safe_name
filename = os.path.basename(dist_path)
_, local_version, local_package_type = parse_filename(filename,
project_hint=name)
remote_dists = client.get_project_files(name)
for remote_dist in remote_dists:
if (versions_equal(local_version, remote_dist.version)
and remote_dist.package_type == local_package_type):
break
else:
return hash_file(dist_path, hashlib.md5), None
try:
algo, remote_hash = url_hash_fragment(remote_dist.url)
except Exception as exc:
raise UnsupportedAPIError("API doesn't support hashes.") from exc
if algo not in hashlib.algorithms_guaranteed:
raise UnsupportedAPIError(f"API returned an unsupported hash: {algo}.")
hashlib_cls = getattr(hashlib, algo)
local_hash = hash_file(dist_path, hashlib_cls)
return local_hash, remote_hash
def register(register_settings, package):
repository_url = register_settings.repository_config['repository']
print("Registering package to {0}".format(repository_url))
repository = register_settings.create_repository()
if not os.path.exists(package):
raise exc.PackageNotFound(
'"{0}" does not exist on the file system.'.format(package))
resp = repository.register(
PackageFile.from_filename(package, register_settings.comment))
repository.close()
if resp.is_redirect:
raise exc.RedirectDetected(
('"{0}" attempted to redirect to "{1}" during registration.'
' Aborting...').format(repository_url, resp.headers["location"]))
resp.raise_for_status()
def register(register_settings, package):
repository_url = register_settings.repository_config['repository']
print(f"Registering package to {repository_url}")
repository = register_settings.create_repository()
if not os.path.exists(package):
raise exceptions.PackageNotFound(
f'"{package}" does not exist on the file system.')
resp = repository.register(
PackageFile.from_filename(package, register_settings.comment))
repository.close()
if resp.is_redirect:
raise exceptions.RedirectDetected.from_args(
repository_url,
resp.headers["location"],
)
resp.raise_for_status()
def register(register_settings, package):
repository_url = register_settings.repository_config['repository']
print("Registering package to {}".format(repository_url))
repository = register_settings.create_repository()
if not os.path.exists(package):
raise exceptions.PackageNotFound(
'"{}" does not exist on the file system.'.format(package)
)
resp = repository.register(
PackageFile.from_filename(package, register_settings.comment)
)
repository.close()
if resp.is_redirect:
raise exceptions.RedirectDetected(
('"{0}" attempted to redirect to "{1}" during registration.'
' Aborting...').format(repository_url,
resp.headers["location"]))
resp.raise_for_status()
def register(self, poet):
"""
Register a package represented by a Poet instance.
:param poet: The Poet instance representing the package.
:type poet: poet.poet.Poet
"""
package = os.path.join(poet.base_dir, 'dist', poet.archive)
if not os.path.exists(package):
raise PackageNotFound(
'"{0}" does not exist on the file system.'.format(package))
resp = self._repository.register(
PackageFile.from_filename(package, None))
self._repository.close()
if resp.is_redirect:
raise RedirectDetected(
('"{0}" attempted to redirect to "{1}" during registration.'
' Aborting...').format(self._repository.url,
resp.headers["location"]))
resp.raise_for_status()
def upload(dists, repository, sign, identity, username, password, comment,
sign_with, config_file, skip_existing, cert, client_cert,
repository_url):
# Check that a nonsensical option wasn't given
if not sign and identity:
raise ValueError("sign must be given along with identity")
dists = find_dists(dists)
# Determine if the user has passed in pre-signed distributions
signatures = dict(
(os.path.basename(d), d) for d in dists if d.endswith(".asc")
)
uploads = [i for i in dists if not i.endswith(".asc")]
config = utils.get_repository_from_config(
config_file,
repository,
repository_url,
)
config["repository"] = utils.normalize_repository_url(
config["repository"]
)
print("Uploading distributions to {0}".format(config["repository"]))
username = utils.get_username(username, config)
password = utils.get_password(
config["repository"], username, password, config,
)
ca_cert = utils.get_cacert(cert, config)
client_cert = utils.get_clientcert(client_cert, config)
repository = Repository(config["repository"], username, password)
repository.set_certificate_authority(ca_cert)
repository.set_client_certificate(client_cert)
for filename in uploads:
package = PackageFile.from_filename(filename, comment)
skip_message = (
" Skipping {0} because it appears to already exist".format(
package.basefilename)
)
# Note: The skip_existing check *needs* to be first, because otherwise
# we're going to generate extra HTTP requests against a hardcoded
# URL for no reason.
if skip_existing and repository.package_is_uploaded(package):
print(skip_message)
continue
signed_name = package.signed_basefilename
if signed_name in signatures:
package.add_gpg_signature(signatures[signed_name], signed_name)
elif sign:
package.sign(sign_with, identity)
resp = repository.upload(package)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise exc.RedirectDetected(
('"{0}" attempted to redirect to "{1}" during upload.'
' Aborting...').format(config["repository"],
resp.headers["location"]))
if skip_upload(resp, skip_existing, package):
print(skip_message)
continue
utils.check_status_code(resp)
# Bug 28. Try to silence a ResourceWarning by clearing the connection
# pool.
repository.close()
def upload(upload_settings, dists):
dists = _find_dists(dists)
# Determine if the user has passed in pre-signed distributions
signatures = {os.path.basename(d): d for d in dists if d.endswith(".asc")}
uploads = [i for i in dists if not i.endswith(".asc")]
upload_settings.check_repository_url()
repository_url = upload_settings.repository_config['repository']
print(f"Uploading distributions to {repository_url}")
repository = upload_settings.create_repository()
uploaded_packages = []
for filename in uploads:
package = PackageFile.from_filename(filename, upload_settings.comment)
skip_message = (
" Skipping {} because it appears to already exist".format(
package.basefilename))
# Note: The skip_existing check *needs* to be first, because otherwise
# we're going to generate extra HTTP requests against a hardcoded
# URL for no reason.
if (upload_settings.skip_existing
and repository.package_is_uploaded(package)):
print(skip_message)
continue
signed_name = package.signed_basefilename
if signed_name in signatures:
package.add_gpg_signature(signatures[signed_name], signed_name)
elif upload_settings.sign:
package.sign(upload_settings.sign_with, upload_settings.identity)
# Suppress TLS verification warning on trusted custom certs
with warnings.catch_warnings():
warnings.simplefilter("ignore")
resp = repository.upload(package)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise exceptions.RedirectDetected.from_args(
repository_url,
resp.headers["location"],
)
if skip_upload(resp, upload_settings.skip_existing, package):
print(skip_message)
continue
utils.check_status_code(resp, upload_settings.verbose)
uploaded_packages.append(package)
release_urls = repository.release_urls(uploaded_packages)
if release_urls:
print('\nView at:')
for url in release_urls:
print(url)
# Bug 28. Try to silence a ResourceWarning by clearing the connection
# pool.
repository.close()
def upload(dists, repository, sign, identity, username, password, comment,
sign_with, config_file, skip_existing, cert, client_cert,
repository_url):
# Check that a nonsensical option wasn't given
if not sign and identity:
raise ValueError("sign must be given along with identity")
dists = find_dists(dists)
# Determine if the user has passed in pre-signed distributions
signatures = dict(
(os.path.basename(d), d) for d in dists if d.endswith(".asc")
)
uploads = [i for i in dists if not i.endswith(".asc")]
config = utils.get_repository_from_config(
config_file,
repository,
repository_url,
)
config["repository"] = utils.normalize_repository_url(
config["repository"]
)
print("Uploading distributions to {0}".format(config["repository"]))
if config["repository"].startswith((LEGACY_PYPI, LEGACY_TEST_PYPI)):
raise exc.UploadToDeprecatedPyPIDetected(
"You're trying to upload to the legacy PyPI site '{0}'. "
"Uploading to those sites is deprecated. \n "
"The new sites are pypi.org and test.pypi.org. Try using "
"{1} (or {2}) to upload your packages instead. "
"These are the default URLs for Twine now. \n More at "
"https://packaging.python.org/guides/migrating-to-pypi-org/ "
".".format(
config["repository"],
utils.DEFAULT_REPOSITORY,
utils.TEST_REPOSITORY
)
)
username = utils.get_username(username, config)
password = utils.get_password(
config["repository"], username, password, config,
)
ca_cert = utils.get_cacert(cert, config)
client_cert = utils.get_clientcert(client_cert, config)
repository = Repository(config["repository"], username, password)
repository.set_certificate_authority(ca_cert)
repository.set_client_certificate(client_cert)
for filename in uploads:
package = PackageFile.from_filename(filename, comment)
skip_message = (
" Skipping {0} because it appears to already exist".format(
package.basefilename)
)
# Note: The skip_existing check *needs* to be first, because otherwise
# we're going to generate extra HTTP requests against a hardcoded
# URL for no reason.
if skip_existing and repository.package_is_uploaded(package):
print(skip_message)
continue
signed_name = package.signed_basefilename
if signed_name in signatures:
package.add_gpg_signature(signatures[signed_name], signed_name)
elif sign:
package.sign(sign_with, identity)
resp = repository.upload(package)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise exc.RedirectDetected(
('"{0}" attempted to redirect to "{1}" during upload.'
' Aborting...').format(config["repository"],
resp.headers["location"]))
if skip_upload(resp, skip_existing, package):
print(skip_message)
continue
utils.check_status_code(resp)
# Bug 28. Try to silence a ResourceWarning by clearing the connection
# pool.
repository.close()