def test_update_description_html(monkeypatch, db_request):
current_version = "24.0"
previous_version = "23.0"
monkeypatch.setattr(readme, "renderer_version", lambda: current_version)
descriptions = [
DescriptionFactory.create(html="rendered",
rendered_by=current_version),
DescriptionFactory.create(html="not this one",
rendered_by=previous_version),
DescriptionFactory.create(html="",
rendered_by=""), # Initial migration state
]
update_description_html(db_request)
assert set(
db_request.db.query(Description.raw, Description.html,
Description.rendered_by).all()) == {
(descriptions[0].raw, "rendered",
current_version),
(descriptions[1].raw,
readme.render(descriptions[1].raw),
current_version),
(descriptions[2].raw,
readme.render(descriptions[2].raw),
current_version),
}
def activity(self):
"""
Render raw activity markdown as HTML
"""
if not self.activity_markdown:
return ""
return readme.render(self.activity_markdown, "text/markdown")
def release_detail(release, request):
project = release.project
# Check if the requested version is equivalent but not exactly the same as
# the release's version. Use `.get` because this view is used by
# `project_detail` and there may not be a version.
#
# This also handles the case where both the version and the project name
# need adjusted, and handles it in a single redirect.
if release.version != request.matchdict.get("version", release.version):
return HTTPMovedPermanently(
request.current_route_path(name=project.name, version=release.version)
)
# It's possible that the requested version was correct (or not provided),
# but we still need to adjust the project name.
if project.name != request.matchdict.get("name", project.name):
return HTTPMovedPermanently(request.current_route_path(name=project.name))
# Render the release description.
description = readme.render(release.description, release.description_content_type)
# Get all of the maintainers for this project.
maintainers = [
r.user
for r in (
request.db.query(Role)
.join(User)
.filter(Role.project == project)
.distinct(User.username)
.order_by(User.username)
.all()
)
]
# Get the license from both the `Classifier` and `License` metadata fields
license_classifiers = ", ".join(
c.split(" :: ")[-1] for c in release.classifiers if c.startswith("License")
)
# Make a best effort when the entire license text is given by using the
# first line only.
short_license = release.license.split("\n")[0] if release.license else None
if license_classifiers and short_license:
license = f"{license_classifiers} ({short_license})"
else:
license = license_classifiers or short_license or None
return {
"project": project,
"release": release,
"description": description,
"files": release.files.all(),
"latest_version": project.latest_version,
"all_versions": project.all_versions,
"maintainers": maintainers,
"license": license,
}
def release_detail(release, request):
project = release.project
# Check if the requested version is equivalent but not exactly the same as
# the release's version. Use `.get` because this view is used by
# `project_detail` and there may not be a version.
#
# This also handles the case where both the version and the project name
# need adjusted, and handles it in a single redirect.
if release.version != request.matchdict.get("version", release.version):
return HTTPMovedPermanently(
request.current_route_path(name=project.name, version=release.version)
)
# It's possible that the requested version was correct (or not provided),
# but we still need to adjust the project name.
if project.name != request.matchdict.get("name", project.name):
return HTTPMovedPermanently(request.current_route_path(name=project.name))
# Render the release description.
description = readme.render(release.description, release.description_content_type)
# Get all of the maintainers for this project.
maintainers = [
r.user
for r in (
request.db.query(Role)
.join(User)
.filter(Role.project == project)
.distinct(User.username)
.order_by(User.username)
.all()
)
]
# Get the license from both the `Classifier` and `License` metadata fields
license_classifiers = ", ".join(
c.split(" :: ")[-1] for c in release.classifiers if c.startswith("License")
)
# Make a best effort when the entire license text is given by using the
# first line only.
short_license = release.license.split("\n")[0] if release.license else None
if license_classifiers and short_license:
license = f"{license_classifiers} ({short_license})"
else:
license = license_classifiers or short_license or None
return {
"project": project,
"release": release,
"description": description,
"files": release.files.all(),
"latest_version": project.latest_version,
"all_versions": project.all_versions,
"maintainers": maintainers,
"license": license,
}
class DescriptionFactory(WarehouseFactory):
class Meta:
model = Description
id = factory.Faker("uuid4", cast_to=None)
raw = factory.Faker("paragraph")
html = factory.LazyAttribute(lambda o: readme.render(o.raw))
rendered_by = factory.LazyAttribute(lambda o: readme.renderer_version())
class DescriptionFactory(WarehouseFactory):
class Meta:
model = Description
id = factory.LazyFunction(uuid.uuid4)
raw = factory.fuzzy.FuzzyText(length=100)
html = factory.LazyAttribute(lambda o: readme.render(o.raw))
rendered_by = factory.LazyAttribute(lambda o: readme.renderer_version())
def update_description_html(request):
renderer_version = readme.renderer_version()
descriptions = (request.db.query(Description).filter(
Description.rendered_by != renderer_version).yield_per(100).limit(500))
for description in descriptions:
description.html = readme.render(description.raw,
description.content_type)
description.rendered_by = renderer_version
def update_description_html(request):
renderer_version = readme.renderer_version()
descriptions = (
request.db.query(Description)
.filter(Description.rendered_by != renderer_version)
.yield_per(100)
.limit(500)
)
for description in descriptions:
description.html = readme.render(description.raw, description.content_type)
description.rendered_by = renderer_version
def test_update_description_html(monkeypatch, db_request):
current_version = "24.0"
previous_version = "23.0"
monkeypatch.setattr(readme, "renderer_version", lambda: current_version)
descriptions = [
DescriptionFactory.create(html="rendered", rendered_by=current_version),
DescriptionFactory.create(html="not this one", rendered_by=previous_version),
DescriptionFactory.create(html="", rendered_by=""), # Initial migration state
]
update_description_html(db_request)
assert set(
db_request.db.query(
Description.raw, Description.html, Description.rendered_by
).all()
) == {
(descriptions[0].raw, "rendered", current_version),
(descriptions[1].raw, readme.render(descriptions[1].raw), current_version),
(descriptions[2].raw, readme.render(descriptions[2].raw), current_version),
}
def file_upload(request):
# If we're in read-only mode, let upload clients know
if request.flags.enabled("read-only"):
raise _exc_with_message(
HTTPForbidden, "Read-only mode: Uploads are temporarily disabled"
)
# Before we do anything, if there isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(
HTTPForbidden, "Invalid or non-existent authentication information."
)
# Ensure that user has a verified, primary email address. This should both
# reduce the ease of spam account creation and activty, as well as act as
# a forcing function for https://github.com/pypa/warehouse/issues/3632.
# TODO: Once https://github.com/pypa/warehouse/issues/3632 has been solved,
# we might consider a different condition, possibly looking at
# User.is_active instead.
if not (request.user.primary_email and request.user.primary_email.verified):
raise _exc_with_message(
HTTPBadRequest,
(
"User {!r} does not have a verified primary email address. "
"Please add a verified primary email before attempting to "
"upload to PyPI. See {project_help} for more information."
"for more information."
).format(
request.user.username,
project_help=request.help_url(_anchor="verified-email"),
),
) from None
# Do some cleanup of the various form fields
for key in list(request.POST):
value = request.POST.get(key)
if isinstance(value, str):
# distutils "helpfully" substitutes unknown, but "required" values
# with the string "UNKNOWN". This is basically never what anyone
# actually wants so we'll just go ahead and delete anything whose
# value is UNKNOWN.
if value.strip() == "UNKNOWN":
del request.POST[key]
# Escape NUL characters, which psycopg doesn't like
if "\x00" in value:
request.POST[key] = value.replace("\x00", "\\x00")
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Check if any fields were supplied as a tuple and have become a
# FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a
# FieldStorage, however.
# ref: https://github.com/pypa/warehouse/issues/2185
# ref: https://github.com/pypa/warehouse/issues/2491
for field in set(request.POST) - {"content", "gpg_signature"}:
values = request.POST.getall(field)
if any(isinstance(value, FieldStorage) for value in values):
raise _exc_with_message(HTTPBadRequest, f"{field}: Should not be a tuple.")
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
# Add a validator for deprecated classifiers
form.classifiers.validators.append(_no_deprecated_classifiers(request))
form.classifiers.choices = [(c.classifier, c.classifier) for c in all_classifiers]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
if field_name in form:
field = form[field_name]
if field.description and isinstance(field, wtforms.StringField):
error_message = (
"{value!r} is an invalid value for {field}. ".format(
value=field.data, field=field.description
)
+ "Error: {} ".format(form.errors[field_name][0])
+ "See "
"https://packaging.python.org/specifications/core-metadata"
)
else:
error_message = "Invalid value for {field}. Error: {msgs[0]}".format(
field=field_name, msgs=form.errors[field_name]
)
else:
error_message = "Error: {}".format(form.errors[field_name][0])
raise _exc_with_message(HTTPBadRequest, error_message)
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(HTTPBadRequest, "Upload payload does not have a file.")
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (
request.db.query(Project)
.filter(
Project.normalized_name == func.normalize_pep426_name(form.name.data)
)
.one()
)
except NoResultFound:
# Check for AdminFlag set by a PyPI Administrator disabling new project
# registration, reasons for this include Spammers, security
# vulnerabilities, or just wanting to be lazy and not worry ;)
if request.flags.enabled("disallow-new-project-registration"):
raise _exc_with_message(
HTTPForbidden,
(
"New project registration temporarily disabled. "
"See {projecthelp} for details"
).format(projecthelp=request.help_url(_anchor="admin-intervention")),
) from None
# Before we create the project, we're going to check our blacklist to
# see if this project is even allowed to be registered. If it is not,
# then we're going to deny the request to create this project.
if request.db.query(
exists().where(
BlacklistedProject.name == func.normalize_pep426_name(form.name.data)
)
).scalar():
raise _exc_with_message(
HTTPBadRequest,
(
"The name {name!r} isn't allowed. "
"See {projecthelp} "
"for more information."
).format(
name=form.name.data,
projecthelp=request.help_url(_anchor="project-name"),
),
) from None
# Also check for collisions with Python Standard Library modules.
if packaging.utils.canonicalize_name(form.name.data) in STDLIB_PROHIBITTED:
raise _exc_with_message(
HTTPBadRequest,
(
"The name {name!r} isn't allowed (conflict with Python "
"Standard Library module name). See "
"{projecthelp} for more information."
).format(
name=form.name.data,
projecthelp=request.help_url(_anchor="project-name"),
),
) from None
# The project doesn't exist in our database, so first we'll check for
# projects with a similar name
squattees = (
request.db.query(Project)
.filter(
func.levenshtein(
Project.normalized_name, func.normalize_pep426_name(form.name.data)
)
<= 2
)
.all()
)
# Next we'll create the project
project = Project(name=form.name.data)
request.db.add(project)
# Now that the project exists, add any squats which it is the squatter for
for squattee in squattees:
request.db.add(Squat(squatter=project, squattee=squattee))
# Then we'll add a role setting the current user as the "Owner" of the
# project.
request.db.add(Role(user=request.user, project=project, role_name="Owner"))
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=project.name,
action="create",
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
request.db.add(
JournalEntry(
name=project.name,
action="add Owner {}".format(request.user.username),
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(
HTTPForbidden,
(
"The user '{0}' isn't allowed to upload to project '{1}'. "
"See {2} for more information."
).format(
request.user.username,
project.name,
request.help_url(_anchor="project-name"),
),
)
# Update name if it differs but is still equivalent. We don't need to check if
# they are equivalent when normalized because that's already been done when we
# queried for the project.
if project.name != form.name.data:
project.name = form.name.data
# Render our description so we can save from having to render this data every time
# we load a project description page.
rendered = None
if form.description.data:
description_content_type = form.description_content_type.data
if not description_content_type:
description_content_type = "text/x-rst"
rendered = readme.render(
form.description.data, description_content_type, use_fallback=False
)
# Uploading should prevent broken rendered descriptions.
if rendered is None:
if form.description_content_type.data:
message = (
"The description failed to render "
"for '{description_content_type}'."
).format(description_content_type=description_content_type)
else:
message = (
"The description failed to render "
"in the default format of reStructuredText."
)
raise _exc_with_message(
HTTPBadRequest,
"{message} See {projecthelp} for more information.".format(
message=message,
projecthelp=request.help_url(_anchor="description-content-type"),
),
) from None
try:
canonical_version = packaging.utils.canonicalize_version(form.version.data)
release = (
request.db.query(Release)
.filter(
(Release.project == project)
& (Release.canonical_version == canonical_version)
)
.one()
)
except MultipleResultsFound:
# There are multiple releases of this project which have the same
# canonical version that were uploaded before we checked for
# canonical version equivalence, so return the exact match instead
release = (
request.db.query(Release)
.filter(
(Release.project == project) & (Release.version == form.version.data)
)
.one()
)
except NoResultFound:
release = Release(
project=project,
_classifiers=[
c for c in all_classifiers if c.classifier in form.classifiers.data
],
dependencies=list(
_construct_dependencies(
form,
{
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
},
)
),
canonical_version=canonical_version,
description=Description(
content_type=form.description_content_type.data,
raw=form.description.data or "",
html=rendered or "",
rendered_by=readme.renderer_version(),
),
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary",
"license",
"author",
"author_email",
"maintainer",
"maintainer_email",
"keywords",
"platform",
"home_page",
"download_url",
"requires_python",
}
},
uploader=request.user,
uploaded_via=request.user_agent,
)
request.db.add(release)
# TODO: This should be handled by some sort of database trigger or
# a SQLAlchemy hook or the like instead of doing it inline in
# this view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="new release",
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = (
request.db.query(Release)
.filter(Release.project == project)
.options(orm.load_only(Release._pypi_ordering))
.all()
)
for i, r in enumerate(
sorted(releases, key=lambda x: packaging.version.parse(x.version))
):
r._pypi_ordering = i
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain any path separators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(
HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name."
)
# Make sure the filename ends with an allowed extension.
if _dist_file_regexes[project.allow_legacy_files].search(filename) is None:
raise _exc_with_message(
HTTPBadRequest,
"Invalid file extension: Use .egg, .tar.gz, .whl or .zip "
"extension. (https://www.python.org/dev/peps/pep-0527)",
)
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest,
"Start filename for {!r} with {!r}.".format(project.name, prefix),
)
# Check the content type of what is being uploaded
if not request.POST["content"].type or request.POST["content"].type.startswith(
"image/"
):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Ensure that the package filetype is allowed.
# TODO: Once PEP 527 is completely implemented we should be able to delete
# this and just move it into the form itself.
if not project.allow_legacy_files and form.filetype.data not in {
"sdist",
"bdist_wheel",
"bdist_egg",
}:
raise _exc_with_message(HTTPBadRequest, "Unknown type of file.")
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
temporary_filename = os.path.join(tmpdir, filename)
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(temporary_filename, "wb") as fp:
file_size = 0
file_hashes = {
"md5": hashlib.md5(),
"sha256": hashlib.sha256(),
"blake2_256": hashlib.blake2b(digest_size=256 // 8),
}
for chunk in iter(lambda: request.POST["content"].file.read(8096), b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(
HTTPBadRequest,
"File too large. "
+ "Limit for project {name!r} is {limit} MB. ".format(
name=project.name, limit=file_size_limit // (1024 * 1024)
)
+ "See "
+ request.help_url(_anchor="file-size-limit"),
)
fp.write(chunk)
for hasher in file_hashes.values():
hasher.update(chunk)
# Take our hash functions and compute the final hashes for them now.
file_hashes = {k: h.hexdigest().lower() for k, h in file_hashes.items()}
# Actually verify the digests that we've gotten. We're going to use
# hmac.compare_digest even though we probably don't actually need to
# because it's better safe than sorry. In the case of multiple digests
# we expect them all to be given.
if not all(
[
hmac.compare_digest(
getattr(form, "{}_digest".format(digest_name)).data.lower(),
digest_value,
)
for digest_name, digest_value in file_hashes.items()
if getattr(form, "{}_digest".format(digest_name)).data
]
):
raise _exc_with_message(
HTTPBadRequest,
"The digest supplied does not match a digest calculated "
"from the uploaded file.",
)
# Check to see if the file that was uploaded exists already or not.
is_duplicate = _is_duplicate_file(request.db, filename, file_hashes)
if is_duplicate:
return Response()
elif is_duplicate is not None:
raise _exc_with_message(
HTTPBadRequest,
# Note: Changing this error message to something that doesn't
# start with "File already exists" will break the
# --skip-existing functionality in twine
# ref: https://github.com/pypa/warehouse/issues/3482
# ref: https://github.com/pypa/twine/issues/332
"File already exists. See "
+ request.help_url(_anchor="file-name-reuse"),
)
# Check to see if the file that was uploaded exists in our filename log
if request.db.query(
request.db.query(Filename).filter(Filename.filename == filename).exists()
).scalar():
raise _exc_with_message(
HTTPBadRequest,
"This filename has already been used, use a "
"different version. "
"See " + request.help_url(_anchor="file-name-reuse"),
)
# Check to see if uploading this file would create a duplicate sdist
# for the current release.
if (
form.filetype.data == "sdist"
and request.db.query(
request.db.query(File)
.filter((File.release == release) & (File.packagetype == "sdist"))
.exists()
).scalar()
):
raise _exc_with_message(
HTTPBadRequest, "Only one sdist may be uploaded per release."
)
# Check the file to make sure it is a valid distribution file.
if not _is_valid_dist_file(temporary_filename, form.filetype.data):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
for plat in plats:
if not _valid_platform_tag(plat):
raise _exc_with_message(
HTTPBadRequest,
"Binary wheel '{filename}' has an unsupported "
"platform tag '{plat}'.".format(filename=filename, plat=plat),
)
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(
lambda: request.POST["gpg_signature"].file.read(8096), b""
):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(HTTPBadRequest, "Signature too large.")
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(
HTTPBadRequest, "PGP signature isn't ASCII armored."
)
else:
has_signature = False
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=file_hashes["md5"],
sha256_digest=file_hashes["sha256"],
blake2_256_digest=file_hashes["blake2_256"],
# Figure out what our filepath is going to be, we're going to use a
# directory structure based on the hash of the file contents. This
# will ensure that the contents of the file cannot change without
# it also changing the path that the file is saved too.
path="/".join(
[
file_hashes[PATH_HASHER][:2],
file_hashes[PATH_HASHER][2:4],
file_hashes[PATH_HASHER][4:],
filename,
]
),
uploaded_via=request.user_agent,
)
request.db.add(file_)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="add {python_version} file {filename}".format(
python_version=file_.python_version, filename=file_.filename
),
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# committed.
storage = request.find_service(IFileStorage)
storage.store(
file_.path,
os.path.join(tmpdir, filename),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
if has_signature:
storage.store(
file_.pgp_path,
os.path.join(tmpdir, filename + ".asc"),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
return Response()
def test_can_render_missing_content_type():
result = readme.render("raw thing")
assert result == "<p>raw thing</p>\n"
def test_can_render_markdown():
result = readme.render("raw thing", "text/markdown")
assert result == "<p>raw thing</p>\n"
def test_can_render_plaintext():
result = readme.render("raw thing", "text/plain")
assert result == "raw thing"
def test_cant_render_rst():
result = readme.render("raw `<thing", "text/x-rst")
assert result == "raw `<thing"
def test_can_render_rst():
result = readme.render("raw thing", "text/x-rst")
assert result == "<p>raw thing</p>\n"
def test_render_with_none():
result = readme.render(None)
assert result is None
def file_upload(request):
# If we're in read-only mode, let upload clients know
if request.flags.enabled("read-only"):
raise _exc_with_message(
HTTPForbidden, "Read-only mode: Uploads are temporarily disabled")
# Log an attempt to upload
metrics = request.find_service(IMetricsService, context=None)
metrics.increment("warehouse.upload.attempt")
# Before we do anything, if there isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(
HTTPForbidden,
"Invalid or non-existent authentication information.")
# Ensure that user has a verified, primary email address. This should both
# reduce the ease of spam account creation and activity, as well as act as
# a forcing function for https://github.com/pypa/warehouse/issues/3632.
# TODO: Once https://github.com/pypa/warehouse/issues/3632 has been solved,
# we might consider a different condition, possibly looking at
# User.is_active instead.
if not (request.user.primary_email
and request.user.primary_email.verified):
raise _exc_with_message(
HTTPBadRequest,
("User {!r} does not have a verified primary email address. "
"Please add a verified primary email before attempting to "
"upload to PyPI. See {project_help} for more information."
"for more information.").format(
request.user.username,
project_help=request.help_url(_anchor="verified-email"),
),
) from None
# Do some cleanup of the various form fields
for key in list(request.POST):
value = request.POST.get(key)
if isinstance(value, str):
# distutils "helpfully" substitutes unknown, but "required" values
# with the string "UNKNOWN". This is basically never what anyone
# actually wants so we'll just go ahead and delete anything whose
# value is UNKNOWN.
if value.strip() == "UNKNOWN":
del request.POST[key]
# Escape NUL characters, which psycopg doesn't like
if "\x00" in value:
request.POST[key] = value.replace("\x00", "\\x00")
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Check if any fields were supplied as a tuple and have become a
# FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a
# FieldStorage, however.
# ref: https://github.com/pypa/warehouse/issues/2185
# ref: https://github.com/pypa/warehouse/issues/2491
for field in set(request.POST) - {"content", "gpg_signature"}:
values = request.POST.getall(field)
if any(isinstance(value, FieldStorage) for value in values):
raise _exc_with_message(HTTPBadRequest,
f"{field}: Should not be a tuple.")
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
# Add a validator for deprecated classifiers
form.classifiers.validators.append(_no_deprecated_classifiers(request))
form.classifiers.choices = [(c.classifier, c.classifier)
for c in all_classifiers]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
if field_name in form:
field = form[field_name]
if field.description and isinstance(field, wtforms.StringField):
error_message = (
"{value!r} is an invalid value for {field}. ".format(
value=field.data, field=field.description) +
"Error: {} ".format(form.errors[field_name][0]) + "See "
"https://packaging.python.org/specifications/core-metadata"
)
else:
error_message = "Invalid value for {field}. Error: {msgs[0]}".format(
field=field_name, msgs=form.errors[field_name])
else:
error_message = "Error: {}".format(form.errors[field_name][0])
raise _exc_with_message(HTTPBadRequest, error_message)
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(HTTPBadRequest,
"Upload payload does not have a file.")
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
form.name.data)).one())
except NoResultFound:
# Check for AdminFlag set by a PyPI Administrator disabling new project
# registration, reasons for this include Spammers, security
# vulnerabilities, or just wanting to be lazy and not worry ;)
if request.flags.enabled("disallow-new-project-registration"):
raise _exc_with_message(
HTTPForbidden,
("New project registration temporarily disabled. "
"See {projecthelp} for details").format(
projecthelp=request.help_url(
_anchor="admin-intervention")),
) from None
# Before we create the project, we're going to check our blacklist to
# see if this project is even allowed to be registered. If it is not,
# then we're going to deny the request to create this project.
if request.db.query(exists().where(
BlacklistedProject.name == func.normalize_pep426_name(
form.name.data))).scalar():
raise _exc_with_message(
HTTPBadRequest,
("The name {name!r} isn't allowed. "
"See {projecthelp} "
"for more information.").format(
name=form.name.data,
projecthelp=request.help_url(_anchor="project-name"),
),
) from None
# Also check for collisions with Python Standard Library modules.
if packaging.utils.canonicalize_name(
form.name.data) in STDLIB_PROHIBITTED:
raise _exc_with_message(
HTTPBadRequest,
("The name {name!r} isn't allowed (conflict with Python "
"Standard Library module name). See "
"{projecthelp} for more information.").format(
name=form.name.data,
projecthelp=request.help_url(_anchor="project-name"),
),
) from None
# The project doesn't exist in our database, so first we'll check for
# projects with a similar name
squattees = (request.db.query(Project).filter(
func.levenshtein(Project.normalized_name,
func.normalize_pep426_name(form.name.data)) <= 2).
all())
# Next we'll create the project
project = Project(name=form.name.data)
request.db.add(project)
# Now that the project exists, add any squats which it is the squatter for
for squattee in squattees:
request.db.add(Squat(squatter=project, squattee=squattee))
# Then we'll add a role setting the current user as the "Owner" of the
# project.
request.db.add(
Role(user=request.user, project=project, role_name="Owner"))
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=project.name,
action="create",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.db.add(
JournalEntry(
name=project.name,
action="add Owner {}".format(request.user.username),
submitted_by=request.user,
submitted_from=request.remote_addr,
))
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(
HTTPForbidden,
("The credential associated with user '{0}' "
"isn't allowed to upload to project '{1}'. "
"See {2} for more information.").format(
request.user.username,
project.name,
request.help_url(_anchor="project-name"),
),
)
# Update name if it differs but is still equivalent. We don't need to check if
# they are equivalent when normalized because that's already been done when we
# queried for the project.
if project.name != form.name.data:
project.name = form.name.data
# Render our description so we can save from having to render this data every time
# we load a project description page.
rendered = None
if form.description.data:
description_content_type = form.description_content_type.data
if not description_content_type:
description_content_type = "text/x-rst"
rendered = readme.render(form.description.data,
description_content_type,
use_fallback=False)
# Uploading should prevent broken rendered descriptions.
if rendered is None:
if form.description_content_type.data:
message = (
"The description failed to render "
"for '{description_content_type}'.").format(
description_content_type=description_content_type)
else:
message = ("The description failed to render "
"in the default format of reStructuredText.")
raise _exc_with_message(
HTTPBadRequest,
"{message} See {projecthelp} for more information.".format(
message=message,
projecthelp=request.help_url(
_anchor="description-content-type"),
),
) from None
try:
canonical_version = packaging.utils.canonicalize_version(
form.version.data)
release = (request.db.query(Release).filter(
(Release.project == project)
& (Release.canonical_version == canonical_version)).one())
except MultipleResultsFound:
# There are multiple releases of this project which have the same
# canonical version that were uploaded before we checked for
# canonical version equivalence, so return the exact match instead
release = (request.db.query(
Release).filter((Release.project == project)
& (Release.version == form.version.data)).one())
except NoResultFound:
release = Release(
project=project,
_classifiers=[
c for c in all_classifiers
if c.classifier in form.classifiers.data
],
dependencies=list(
_construct_dependencies(
form,
{
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
},
)),
canonical_version=canonical_version,
description=Description(
content_type=form.description_content_type.data,
raw=form.description.data or "",
html=rendered or "",
rendered_by=readme.renderer_version(),
),
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary",
"license",
"author",
"author_email",
"maintainer",
"maintainer_email",
"keywords",
"platform",
"home_page",
"download_url",
"requires_python",
}
},
uploader=request.user,
uploaded_via=request.user_agent,
)
request.db.add(release)
# TODO: This should be handled by some sort of database trigger or
# a SQLAlchemy hook or the like instead of doing it inline in
# this view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="new release",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = (request.db.query(Release).filter(
Release.project == project).options(
orm.load_only(Release._pypi_ordering)).all())
for i, r in enumerate(
sorted(releases,
key=lambda x: packaging.version.parse(x.version))):
r._pypi_ordering = i
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain any path separators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(
HTTPBadRequest,
"Cannot upload a file with '/' or '\\' in the name.")
# Make sure the filename ends with an allowed extension.
if _dist_file_regexes[project.allow_legacy_files].search(filename) is None:
raise _exc_with_message(
HTTPBadRequest,
"Invalid file extension: Use .egg, .tar.gz, .whl or .zip "
"extension. (https://www.python.org/dev/peps/pep-0527)",
)
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest,
"Start filename for {!r} with {!r}.".format(project.name, prefix),
)
# Check the content type of what is being uploaded
if not request.POST["content"].type or request.POST[
"content"].type.startswith("image/"):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Ensure that the package filetype is allowed.
# TODO: Once PEP 527 is completely implemented we should be able to delete
# this and just move it into the form itself.
if not project.allow_legacy_files and form.filetype.data not in {
"sdist",
"bdist_wheel",
"bdist_egg",
}:
raise _exc_with_message(HTTPBadRequest, "Unknown type of file.")
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
temporary_filename = os.path.join(tmpdir, filename)
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(temporary_filename, "wb") as fp:
file_size = 0
file_hashes = {
"md5": hashlib.md5(),
"sha256": hashlib.sha256(),
"blake2_256": hashlib.blake2b(digest_size=256 // 8),
}
for chunk in iter(lambda: request.POST["content"].file.read(8096),
b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(
HTTPBadRequest,
"File too large. " +
"Limit for project {name!r} is {limit} MB. ".format(
name=project.name,
limit=file_size_limit // (1024 * 1024)) + "See " +
request.help_url(_anchor="file-size-limit"),
)
fp.write(chunk)
for hasher in file_hashes.values():
hasher.update(chunk)
# Take our hash functions and compute the final hashes for them now.
file_hashes = {
k: h.hexdigest().lower()
for k, h in file_hashes.items()
}
# Actually verify the digests that we've gotten. We're going to use
# hmac.compare_digest even though we probably don't actually need to
# because it's better safe than sorry. In the case of multiple digests
# we expect them all to be given.
if not all([
hmac.compare_digest(
getattr(form,
"{}_digest".format(digest_name)).data.lower(),
digest_value,
) for digest_name, digest_value in file_hashes.items()
if getattr(form, "{}_digest".format(digest_name)).data
]):
raise _exc_with_message(
HTTPBadRequest,
"The digest supplied does not match a digest calculated "
"from the uploaded file.",
)
# Check to see if the file that was uploaded exists already or not.
is_duplicate = _is_duplicate_file(request.db, filename, file_hashes)
if is_duplicate:
return Response()
elif is_duplicate is not None:
raise _exc_with_message(
HTTPBadRequest,
# Note: Changing this error message to something that doesn't
# start with "File already exists" will break the
# --skip-existing functionality in twine
# ref: https://github.com/pypa/warehouse/issues/3482
# ref: https://github.com/pypa/twine/issues/332
"File already exists. See " +
request.help_url(_anchor="file-name-reuse"),
)
# Check to see if the file that was uploaded exists in our filename log
if request.db.query(
request.db.query(Filename).filter(
Filename.filename == filename).exists()).scalar():
raise _exc_with_message(
HTTPBadRequest,
"This filename has already been used, use a "
"different version. "
"See " + request.help_url(_anchor="file-name-reuse"),
)
# Check to see if uploading this file would create a duplicate sdist
# for the current release.
if (form.filetype.data == "sdist" and request.db.query(
request.db.query(File).filter((File.release == release) & (
File.packagetype == "sdist")).exists()).scalar()):
raise _exc_with_message(
HTTPBadRequest, "Only one sdist may be uploaded per release.")
# Check the file to make sure it is a valid distribution file.
if not _is_valid_dist_file(temporary_filename, form.filetype.data):
raise _exc_with_message(HTTPBadRequest,
"Invalid distribution file.")
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
for plat in plats:
if not _valid_platform_tag(plat):
raise _exc_with_message(
HTTPBadRequest,
"Binary wheel '{filename}' has an unsupported "
"platform tag '{plat}'.".format(filename=filename,
plat=plat),
)
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(
lambda: request.POST["gpg_signature"].file.read(8096),
b""):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(HTTPBadRequest,
"Signature too large.")
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(
HTTPBadRequest, "PGP signature isn't ASCII armored.")
else:
has_signature = False
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=file_hashes["md5"],
sha256_digest=file_hashes["sha256"],
blake2_256_digest=file_hashes["blake2_256"],
# Figure out what our filepath is going to be, we're going to use a
# directory structure based on the hash of the file contents. This
# will ensure that the contents of the file cannot change without
# it also changing the path that the file is saved too.
path="/".join([
file_hashes[PATH_HASHER][:2],
file_hashes[PATH_HASHER][2:4],
file_hashes[PATH_HASHER][4:],
filename,
]),
uploaded_via=request.user_agent,
)
request.db.add(file_)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="add {python_version} file {filename}".format(
python_version=file_.python_version,
filename=file_.filename),
submitted_by=request.user,
submitted_from=request.remote_addr,
))
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# committed.
storage = request.find_service(IFileStorage)
storage.store(
file_.path,
os.path.join(tmpdir, filename),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
if has_signature:
storage.store(
file_.pgp_path,
os.path.join(tmpdir, filename + ".asc"),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
# Log a successful upload
metrics.increment("warehouse.upload.ok",
tags=[f"filetype:{form.filetype.data}"])
return Response()
def release_detail(release, request):
project = release.project
# Check if the requested version is equivalent but not exactly the same as
# the release's version. Use `.get` because this view is used by
# `project_detail` and there may not be a version.
#
# This also handles the case where both the version and the project name
# need adjusted, and handles it in a single redirect.
if release.version != request.matchdict.get("version", release.version):
return HTTPMovedPermanently(
request.current_route_path(name=project.name,
version=release.version))
# It's possible that the requested version was correct (or not provided),
# but we still need to adjust the project name.
if project.name != request.matchdict.get("name", project.name):
return HTTPMovedPermanently(
request.current_route_path(name=project.name))
# Grab the rendered description if it exists, and if it doesn't, then we will render
# it inline.
# TODO: Remove the fallback to rendering inline and only support displaying the
# already rendered content.
if release.description.html:
description = release.description.html
else:
description = readme.render(release.description.raw,
release.description.content_type)
# Get all of the maintainers for this project.
maintainers = [
r.user for r in (request.db.query(Role).join(User).filter(
Role.project == project).distinct(User.username).order_by(
User.username).all())
]
# Get the license from both the `Classifier` and `License` metadata fields
license_classifiers = ", ".join(
c.split(" :: ")[-1] for c in release.classifiers
if c.startswith("License"))
# Make a best effort when the entire license text is given by using the
# first line only.
short_license = release.license.split("\n")[0] if release.license else None
if license_classifiers and short_license:
license = f"{license_classifiers} ({short_license})"
else:
license = license_classifiers or short_license or None
# We cannot easily sort naturally in SQL, sort here and pass to template
sdists = natsorted(
release.files.filter(File.packagetype == "sdist").all(),
reverse=True,
key=lambda f: f.filename,
)
bdists = natsorted(
release.files.filter(File.packagetype != "sdist").all(),
reverse=True,
key=lambda f: f.filename,
)
return {
"project": project,
"release": release,
"description": description,
"files": sdists + bdists,
"sdists": sdists,
"bdists": bdists,
"latest_version": project.latest_version,
"all_versions": project.all_versions,
"maintainers": maintainers,
"license": license,
}
