def user_delete(request):
user = request.db.query(User).get(request.matchdict["user_id"])
if user.username != request.params.get("username"):
request.session.flash("Wrong confirmation input", queue="error")
return HTTPSeeOther(request.route_path("admin.user.detail", user_id=user.id))
# Delete all the user's projects
projects = request.db.query(Project).filter(
Project.name.in_(
request.db.query(Project.name)
.join(Role.project)
.filter(Role.user == user)
.subquery()
)
)
for project in projects:
request.db.add(
JournalEntry(
name=project.name,
action="remove project",
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
projects.delete(synchronize_session=False)
# Update all journals to point to `deleted-user` instead
deleted_user = request.db.query(User).filter(User.username == "deleted-user").one()
journals = (
request.db.query(JournalEntry)
.options(joinedload("submitted_by"))
.filter(JournalEntry.submitted_by == user)
.all()
)
for journal in journals:
journal.submitted_by = deleted_user
# Delete the user
request.db.delete(user)
request.db.add(
JournalEntry(
name=f"user:{user.username}",
action="nuke user",
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
request.session.flash(f"Nuked user {user.username!r}", queue="success")
return HTTPSeeOther(request.route_path("admin.user.list"))
def delete_project_role(project, request):
# TODO: This view was modified to handle deleting multiple roles for a
# single user and should be updated when fixing GH-2745
roles = (request.db.query(Role).filter(
Role.id.in_(request.POST.getall("role_id")),
Role.project == project).all())
removing_self = any(role.role_name == "Owner" and role.user == request.user
for role in roles)
if not roles:
request.session.flash("Could not find role", queue="error")
elif removing_self:
request.session.flash("Cannot remove yourself as Owner", queue="error")
else:
for role in roles:
request.db.delete(role)
request.db.add(
JournalEntry(
name=project.name,
action=f"remove {role.role_name} {role.user_name}",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.session.flash("Removed role", queue="success")
return HTTPSeeOther(
request.route_path("manage.project.roles", project_name=project.name))
def delete_role(project, request):
confirm = request.POST.get("username")
role_id = request.matchdict.get("role_id")
role = request.db.query(Role).get(role_id)
if not role:
request.session.flash(f"This role no longer exists", queue="error")
raise HTTPSeeOther(
request.route_path("admin.project.detail",
project_name=project.normalized_name))
if not confirm or confirm != role.user.username:
request.session.flash("Confirm the request", queue="error")
raise HTTPSeeOther(
request.route_path("admin.project.detail",
project_name=project.normalized_name))
request.session.flash(
f"Removed '{role.user.username}' as '{role.role_name}' on '{project.name}'",
queue="success",
)
request.db.add(
JournalEntry(
name=project.name,
action=f"remove {role.role_name} {role.user.username}",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.db.delete(role)
return HTTPSeeOther(
request.route_path("admin.project.detail",
project_name=project.normalized_name))
def delete_project_release_file(self):
def _error(message):
self.request.session.flash(message, queue='error')
return HTTPSeeOther(
self.request.route_path(
'manage.project.release',
project_name=self.release.project.name,
version=self.release.version,
)
)
project_name = self.request.POST.get('confirm_project_name')
if not project_name:
return _error("Must confirm the request")
try:
release_file = (
self.request.db.query(File)
.filter(
File.name == self.release.project.name,
File.id == self.request.POST.get('file_id'),
)
.one()
)
except NoResultFound:
return _error('Could not find file')
if project_name != self.release.project.name:
return _error(
"Could not delete file - " +
f"{project_name!r} is not the same as "
f"{self.release.project.name!r}",
)
self.request.db.add(
JournalEntry(
name=self.release.project.name,
action=f"remove file {release_file.filename}",
version=self.release.version,
submitted_by=self.request.user,
submitted_from=self.request.remote_addr,
),
)
self.request.db.delete(release_file)
self.request.session.flash(
f"Successfully deleted file {release_file.filename!r}",
queue="success",
)
return HTTPSeeOther(
self.request.route_path(
'manage.project.release',
project_name=self.release.project.name,
version=self.release.version,
)
)
def delete_project_release_file(self):
def _error(message):
self.request.session.flash(message, queue="error")
return HTTPSeeOther(
self.request.route_path(
"manage.project.release",
project_name=self.release.project.name,
version=self.release.version,
))
project_name = self.request.POST.get("confirm_project_name")
if not project_name:
return _error("Confirm the request")
try:
release_file = (self.request.db.query(File).filter(
File.release == self.release,
File.id == self.request.POST.get("file_id"),
).one())
except NoResultFound:
return _error("Could not find file")
if project_name != self.release.project.name:
return _error("Could not delete file - " +
f"{project_name!r} is not the same as "
f"{self.release.project.name!r}")
self.request.db.add(
JournalEntry(
name=self.release.project.name,
action=f"remove file {release_file.filename}",
version=self.release.version,
submitted_by=self.request.user,
submitted_from=self.request.remote_addr,
))
self.release.project.record_event(
tag="project:release:file:remove",
ip_address=self.request.remote_addr,
additional={
"submitted_by": self.request.user.username,
"canonical_version": self.release.canonical_version,
"filename": release_file.filename,
},
)
self.request.db.delete(release_file)
self.request.session.flash(f"Deleted file {release_file.filename!r}",
queue="success")
return HTTPSeeOther(
self.request.route_path(
"manage.project.release",
project_name=self.release.project.name,
version=self.release.version,
))
def manage_project_roles(project, request, _form_class=CreateRoleForm):
user_service = request.find_service(IUserService, context=None)
form = _form_class(request.POST, user_service=user_service)
if request.method == "POST" and form.validate():
username = form.username.data
role_name = form.role_name.data
userid = user_service.find_userid(username)
user = user_service.get_user(userid)
if (request.db.query(
request.db.query(Role).filter(
Role.user == user,
Role.project == project,
Role.role_name == role_name,
)
.exists()).scalar()):
request.session.flash(
f"User '{username}' already has {role_name} role for project",
queue="error"
)
else:
request.db.add(
Role(user=user, project=project, role_name=form.role_name.data)
)
request.db.add(
JournalEntry(
name=project.name,
action=f"add {role_name} {username}",
submitted_by=request.user,
submitted_from=request.remote_addr,
),
)
request.session.flash(
f"Added collaborator '{form.username.data}'",
queue="success"
)
form = _form_class(user_service=user_service)
roles = (
request.db.query(Role)
.join(User)
.filter(Role.project == project)
.all()
)
# TODO: The following lines are a hack to handle multiple roles for a
# single user and should be removed when fixing GH-2745
roles_by_user = defaultdict(list)
for role in roles:
roles_by_user[role.user.username].append(role)
return {
"project": project,
"roles_by_user": roles_by_user,
"form": form,
}
def delete_project_release(self):
version = self.request.POST.get("confirm_version")
if not version:
self.request.session.flash("Confirm the request", queue="error")
return HTTPSeeOther(
self.request.route_path(
"manage.project.release",
project_name=self.release.project.name,
version=self.release.version,
)
)
if version != self.release.version:
self.request.session.flash(
"Could not delete release - "
+ f"{version!r} is not the same as {self.release.version!r}",
queue="error",
)
return HTTPSeeOther(
self.request.route_path(
"manage.project.release",
project_name=self.release.project.name,
version=self.release.version,
)
)
self.request.db.add(
JournalEntry(
name=self.release.project.name,
action="remove release",
version=self.release.version,
submitted_by=self.request.user,
submitted_from=self.request.remote_addr,
)
)
self.release.project.record_event(
tag="project:release:remove",
ip_address=self.request.remote_addr,
additional={
"submitted_by": self.request.user.username,
"canonical_version": self.release.canonical_version,
},
)
self.request.db.delete(self.release)
self.request.session.flash(
f"Deleted release {self.release.version!r}", queue="success"
)
return HTTPSeeOther(
self.request.route_path(
"manage.project.releases", project_name=self.release.project.name
)
)
def _nuke_user(user, request):
# Delete all the user's projects
projects = request.db.query(Project).filter(
Project.name.in_(
request.db.query(Project.name).join(
Role.project).filter(Role.user == user).subquery()))
for project in projects:
request.db.add(
JournalEntry(
name=project.name,
action="remove project",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
projects.delete(synchronize_session=False)
# Update all journals to point to `deleted-user` instead
deleted_user = request.db.query(User).filter(
User.username == "deleted-user").one()
journals = (request.db.query(JournalEntry).options(
joinedload("submitted_by")).filter(
JournalEntry.submitted_by == user).all())
for journal in journals:
journal.submitted_by = deleted_user
# Prohibit the username
request.db.add(
ProhibitedUserName(name=user.username.lower(),
comment="nuked",
prohibited_by=request.user))
# Delete the user
request.db.delete(user)
request.db.add(
JournalEntry(
name=f"user:{user.username}",
action="nuke user",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
def delete_project_release(self):
version = self.request.POST.get('confirm_version')
if not version:
self.request.session.flash(
"Must confirm the request", queue='error'
)
return HTTPSeeOther(
self.request.route_path(
'manage.project.release',
project_name=self.release.project.name,
version=self.release.version,
)
)
if version != self.release.version:
self.request.session.flash(
"Could not delete release - " +
f"{version!r} is not the same as {self.release.version!r}",
queue="error",
)
return HTTPSeeOther(
self.request.route_path(
'manage.project.release',
project_name=self.release.project.name,
version=self.release.version,
)
)
self.request.db.add(
JournalEntry(
name=self.release.project.name,
action="remove",
version=self.release.version,
submitted_by=self.request.user,
submitted_from=self.request.remote_addr,
),
)
self.request.db.delete(self.release)
self.request.session.flash(
f"Successfully deleted release {self.release.version!r}",
queue="success",
)
return HTTPSeeOther(
self.request.route_path(
'manage.project.releases',
project_name=self.release.project.name,
)
)
def add_role(project, request):
username = request.POST.get("username")
if not username:
request.session.flash("Provide a username", queue="error")
raise HTTPSeeOther(
request.route_path("admin.project.detail",
project_name=project.normalized_name))
try:
user = request.db.query(User).filter(User.username == username).one()
except NoResultFound:
request.session.flash(f"Unknown username '{username}'", queue="error")
raise HTTPSeeOther(
request.route_path("admin.project.detail",
project_name=project.normalized_name))
role_name = request.POST.get("role_name")
if not role_name:
request.session.flash("Provide a role", queue="error")
raise HTTPSeeOther(
request.route_path("admin.project.detail",
project_name=project.normalized_name))
already_there = (request.db.query(Role).filter(
Role.user == user, Role.project == project).count())
if already_there > 0:
request.session.flash(
f"User '{user.username}' already has a role on this project",
queue="error")
raise HTTPSeeOther(
request.route_path("admin.project.detail",
project_name=project.normalized_name))
request.db.add(
JournalEntry(
name=project.name,
action=f"add {role_name} {user.username}",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.db.add(Role(role_name=role_name, user=user, project=project))
request.session.flash(
f"Added '{user.username}' as '{role_name}' on '{project.name}'",
queue="success")
return HTTPSeeOther(
request.route_path("admin.project.detail",
project_name=project.normalized_name))
def destroy_docs(project, request, flash=True):
request.db.add(
JournalEntry(
name=project.name,
action="docdestroy",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.task(remove_documentation).delay(project.name)
project.has_docs = False
if flash:
request.session.flash(f"Deleted docs for project {project.name!r}",
queue="success")
def user_delete(request):
user = request.db.query(User).get(request.matchdict['user_id'])
if user.username != request.params.get('username'):
print(user.username)
print(request.params.get('username'))
request.session.flash(f'Wrong confirmation input.', queue='error')
return HTTPSeeOther(
request.route_path('admin.user.detail', user_id=user.id)
)
# Delete projects one by one so they are purged from the cache
for project in user.projects:
remove_project(project, request, flash=False)
# Update all journals to point to `deleted-user` instead
deleted_user = (
request.db.query(User)
.filter(User.username == 'deleted-user')
.one()
)
journals = (
request.db.query(JournalEntry)
.filter(JournalEntry.submitted_by == user)
.all()
)
for journal in journals:
journal.submitted_by = deleted_user
# Delete the user
request.db.delete(user)
request.db.add(
JournalEntry(
name=f'user:{user.username}',
action=f'nuke user',
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
request.session.flash(f'Nuked user {user.username!r}.', queue='success')
return HTTPSeeOther(request.route_path('admin.user.list'))
def remove_project(project, request, flash=True):
# TODO: We don't actually delete files from the data store. We should add
# some kind of garbage collection at some point.
request.db.add(
JournalEntry(
name=project.name,
action="remove project",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.db.delete(project)
# Flush so we can repeat this multiple times if necessary
request.db.flush()
if flash:
request.session.flash(f"Deleted the project {project.name!r}",
queue="success")
def remove_project(project, request, flash=True):
# TODO: We don't actually delete files from the data store. We should add
# some kind of garbage collection at some point.
request.db.add(
JournalEntry(
name=project.name,
action="remove",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.db.query(Role).filter(Role.project == project).delete()
request.db.query(File).filter(File.name == project.name).delete()
(request.db.query(Dependency).filter(
Dependency.name == project.name).delete())
(request.db.execute(release_classifiers.delete().where(
release_classifiers.c.name == project.name)))
# Load the following objects into the session and individually delete them
# so they are included in `session.deleted` and their cache keys are purged
# Delete releases first, otherwise they will get cascade-deleted by the
# project deletion and won't be purged
for release in (request.db.query(Release).filter(
Release.project == project).all()):
request.db.delete(release)
# Finally, delete the project
request.db.delete(project)
# Flush so we can repeat this multiple times if necessary
request.db.flush()
if flash:
request.session.flash(
f"Successfully deleted the project {project.name!r}.",
queue="success",
)
def user_delete(request):
user = request.db.query(User).get(request.matchdict['user_id'])
if user.username != request.params.get('username'):
request.session.flash(f'Wrong confirmation input', queue='error')
return HTTPSeeOther(
request.route_path('admin.user.detail', user_id=user.id))
# Delete all the user's projects
projects = (request.db.query(Project).filter(
Project.name.in_(
request.db.query(Project.name).join(
Role.project).filter(Role.user == user).subquery())))
projects.delete(synchronize_session=False)
# Update all journals to point to `deleted-user` instead
deleted_user = (request.db.query(User).filter(
User.username == 'deleted-user').one())
journals = (request.db.query(JournalEntry).filter(
JournalEntry.submitted_by == user).all())
for journal in journals:
journal.submitted_by = deleted_user
# Delete the user
request.db.delete(user)
request.db.add(
JournalEntry(
name=f'user:{user.username}',
action=f'nuke user',
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.session.flash(f'Nuked user {user.username!r}', queue='success')
return HTTPSeeOther(request.route_path('admin.user.list'))
def remove_project(project, request):
# TODO: We don't actually delete files from the data store. We should add
# some kind of garbage collection at some point.
request.db.add(
JournalEntry(
name=project.name,
action="remove",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.db.query(Role).filter(Role.project == project).delete()
request.db.query(File).filter(File.name == project.name).delete()
(request.db.query(Dependency).filter(
Dependency.name == project.name).delete())
(request.db.execute(release_classifiers.delete().where(
release_classifiers.c.name == project.name)))
request.db.query(Release).filter(Release.name == project.name).delete()
request.db.query(Project).filter(Project.name == project.name).delete()
request.session.flash(
f"Successfully deleted the project {project.name!r}.",
queue="success",
)
def file_upload(request):
# If we're in read-only mode, let upload clients know
if request.flags.enabled("read-only"):
raise _exc_with_message(
HTTPForbidden, "Read-only mode: Uploads are temporarily disabled")
# Log an attempt to upload
metrics = request.find_service(IMetricsService, context=None)
metrics.increment("warehouse.upload.attempt")
# Before we do anything, if there isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(
HTTPForbidden,
"Invalid or non-existent authentication information.")
# Ensure that user has a verified, primary email address. This should both
# reduce the ease of spam account creation and activity, as well as act as
# a forcing function for https://github.com/pypa/warehouse/issues/3632.
# TODO: Once https://github.com/pypa/warehouse/issues/3632 has been solved,
# we might consider a different condition, possibly looking at
# User.is_active instead.
if not (request.user.primary_email
and request.user.primary_email.verified):
raise _exc_with_message(
HTTPBadRequest,
("User {!r} does not have a verified primary email address. "
"Please add a verified primary email before attempting to "
"upload to PyPI. See {project_help} for more information."
"for more information.").format(
request.user.username,
project_help=request.help_url(_anchor="verified-email"),
),
) from None
# Do some cleanup of the various form fields
for key in list(request.POST):
value = request.POST.get(key)
if isinstance(value, str):
# distutils "helpfully" substitutes unknown, but "required" values
# with the string "UNKNOWN". This is basically never what anyone
# actually wants so we'll just go ahead and delete anything whose
# value is UNKNOWN.
if value.strip() == "UNKNOWN":
del request.POST[key]
# Escape NUL characters, which psycopg doesn't like
if "\x00" in value:
request.POST[key] = value.replace("\x00", "\\x00")
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Check if any fields were supplied as a tuple and have become a
# FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a
# FieldStorage, however.
# ref: https://github.com/pypa/warehouse/issues/2185
# ref: https://github.com/pypa/warehouse/issues/2491
for field in set(request.POST) - {"content", "gpg_signature"}:
values = request.POST.getall(field)
if any(isinstance(value, FieldStorage) for value in values):
raise _exc_with_message(HTTPBadRequest,
f"{field}: Should not be a tuple.")
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
# Add a validator for deprecated classifiers
form.classifiers.validators.append(_no_deprecated_classifiers(request))
form.classifiers.choices = [(c.classifier, c.classifier)
for c in all_classifiers]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
if field_name in form:
field = form[field_name]
if field.description and isinstance(field, wtforms.StringField):
error_message = (
"{value!r} is an invalid value for {field}. ".format(
value=field.data, field=field.description) +
"Error: {} ".format(form.errors[field_name][0]) + "See "
"https://packaging.python.org/specifications/core-metadata"
)
else:
error_message = "Invalid value for {field}. Error: {msgs[0]}".format(
field=field_name, msgs=form.errors[field_name])
else:
error_message = "Error: {}".format(form.errors[field_name][0])
raise _exc_with_message(HTTPBadRequest, error_message)
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(HTTPBadRequest,
"Upload payload does not have a file.")
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
form.name.data)).one())
except NoResultFound:
# Check for AdminFlag set by a PyPI Administrator disabling new project
# registration, reasons for this include Spammers, security
# vulnerabilities, or just wanting to be lazy and not worry ;)
if request.flags.enabled("disallow-new-project-registration"):
raise _exc_with_message(
HTTPForbidden,
("New project registration temporarily disabled. "
"See {projecthelp} for details").format(
projecthelp=request.help_url(
_anchor="admin-intervention")),
) from None
# Before we create the project, we're going to check our blacklist to
# see if this project is even allowed to be registered. If it is not,
# then we're going to deny the request to create this project.
if request.db.query(exists().where(
BlacklistedProject.name == func.normalize_pep426_name(
form.name.data))).scalar():
raise _exc_with_message(
HTTPBadRequest,
("The name {name!r} isn't allowed. "
"See {projecthelp} "
"for more information.").format(
name=form.name.data,
projecthelp=request.help_url(_anchor="project-name"),
),
) from None
# Also check for collisions with Python Standard Library modules.
if packaging.utils.canonicalize_name(
form.name.data) in STDLIB_PROHIBITTED:
raise _exc_with_message(
HTTPBadRequest,
("The name {name!r} isn't allowed (conflict with Python "
"Standard Library module name). See "
"{projecthelp} for more information.").format(
name=form.name.data,
projecthelp=request.help_url(_anchor="project-name"),
),
) from None
# The project doesn't exist in our database, so first we'll check for
# projects with a similar name
squattees = (request.db.query(Project).filter(
func.levenshtein(Project.normalized_name,
func.normalize_pep426_name(form.name.data)) <= 2).
all())
# Next we'll create the project
project = Project(name=form.name.data)
request.db.add(project)
# Now that the project exists, add any squats which it is the squatter for
for squattee in squattees:
request.db.add(Squat(squatter=project, squattee=squattee))
# Then we'll add a role setting the current user as the "Owner" of the
# project.
request.db.add(
Role(user=request.user, project=project, role_name="Owner"))
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=project.name,
action="create",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.db.add(
JournalEntry(
name=project.name,
action="add Owner {}".format(request.user.username),
submitted_by=request.user,
submitted_from=request.remote_addr,
))
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(
HTTPForbidden,
("The credential associated with user '{0}' "
"isn't allowed to upload to project '{1}'. "
"See {2} for more information.").format(
request.user.username,
project.name,
request.help_url(_anchor="project-name"),
),
)
# Update name if it differs but is still equivalent. We don't need to check if
# they are equivalent when normalized because that's already been done when we
# queried for the project.
if project.name != form.name.data:
project.name = form.name.data
# Render our description so we can save from having to render this data every time
# we load a project description page.
rendered = None
if form.description.data:
description_content_type = form.description_content_type.data
if not description_content_type:
description_content_type = "text/x-rst"
rendered = readme.render(form.description.data,
description_content_type,
use_fallback=False)
# Uploading should prevent broken rendered descriptions.
if rendered is None:
if form.description_content_type.data:
message = (
"The description failed to render "
"for '{description_content_type}'.").format(
description_content_type=description_content_type)
else:
message = ("The description failed to render "
"in the default format of reStructuredText.")
raise _exc_with_message(
HTTPBadRequest,
"{message} See {projecthelp} for more information.".format(
message=message,
projecthelp=request.help_url(
_anchor="description-content-type"),
),
) from None
try:
canonical_version = packaging.utils.canonicalize_version(
form.version.data)
release = (request.db.query(Release).filter(
(Release.project == project)
& (Release.canonical_version == canonical_version)).one())
except MultipleResultsFound:
# There are multiple releases of this project which have the same
# canonical version that were uploaded before we checked for
# canonical version equivalence, so return the exact match instead
release = (request.db.query(
Release).filter((Release.project == project)
& (Release.version == form.version.data)).one())
except NoResultFound:
release = Release(
project=project,
_classifiers=[
c for c in all_classifiers
if c.classifier in form.classifiers.data
],
dependencies=list(
_construct_dependencies(
form,
{
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
},
)),
canonical_version=canonical_version,
description=Description(
content_type=form.description_content_type.data,
raw=form.description.data or "",
html=rendered or "",
rendered_by=readme.renderer_version(),
),
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary",
"license",
"author",
"author_email",
"maintainer",
"maintainer_email",
"keywords",
"platform",
"home_page",
"download_url",
"requires_python",
}
},
uploader=request.user,
uploaded_via=request.user_agent,
)
request.db.add(release)
# TODO: This should be handled by some sort of database trigger or
# a SQLAlchemy hook or the like instead of doing it inline in
# this view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="new release",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = (request.db.query(Release).filter(
Release.project == project).options(
orm.load_only(Release._pypi_ordering)).all())
for i, r in enumerate(
sorted(releases,
key=lambda x: packaging.version.parse(x.version))):
r._pypi_ordering = i
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain any path separators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(
HTTPBadRequest,
"Cannot upload a file with '/' or '\\' in the name.")
# Make sure the filename ends with an allowed extension.
if _dist_file_regexes[project.allow_legacy_files].search(filename) is None:
raise _exc_with_message(
HTTPBadRequest,
"Invalid file extension: Use .egg, .tar.gz, .whl or .zip "
"extension. (https://www.python.org/dev/peps/pep-0527)",
)
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest,
"Start filename for {!r} with {!r}.".format(project.name, prefix),
)
# Check the content type of what is being uploaded
if not request.POST["content"].type or request.POST[
"content"].type.startswith("image/"):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Ensure that the package filetype is allowed.
# TODO: Once PEP 527 is completely implemented we should be able to delete
# this and just move it into the form itself.
if not project.allow_legacy_files and form.filetype.data not in {
"sdist",
"bdist_wheel",
"bdist_egg",
}:
raise _exc_with_message(HTTPBadRequest, "Unknown type of file.")
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
temporary_filename = os.path.join(tmpdir, filename)
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(temporary_filename, "wb") as fp:
file_size = 0
file_hashes = {
"md5": hashlib.md5(),
"sha256": hashlib.sha256(),
"blake2_256": hashlib.blake2b(digest_size=256 // 8),
}
for chunk in iter(lambda: request.POST["content"].file.read(8096),
b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(
HTTPBadRequest,
"File too large. " +
"Limit for project {name!r} is {limit} MB. ".format(
name=project.name,
limit=file_size_limit // (1024 * 1024)) + "See " +
request.help_url(_anchor="file-size-limit"),
)
fp.write(chunk)
for hasher in file_hashes.values():
hasher.update(chunk)
# Take our hash functions and compute the final hashes for them now.
file_hashes = {
k: h.hexdigest().lower()
for k, h in file_hashes.items()
}
# Actually verify the digests that we've gotten. We're going to use
# hmac.compare_digest even though we probably don't actually need to
# because it's better safe than sorry. In the case of multiple digests
# we expect them all to be given.
if not all([
hmac.compare_digest(
getattr(form,
"{}_digest".format(digest_name)).data.lower(),
digest_value,
) for digest_name, digest_value in file_hashes.items()
if getattr(form, "{}_digest".format(digest_name)).data
]):
raise _exc_with_message(
HTTPBadRequest,
"The digest supplied does not match a digest calculated "
"from the uploaded file.",
)
# Check to see if the file that was uploaded exists already or not.
is_duplicate = _is_duplicate_file(request.db, filename, file_hashes)
if is_duplicate:
return Response()
elif is_duplicate is not None:
raise _exc_with_message(
HTTPBadRequest,
# Note: Changing this error message to something that doesn't
# start with "File already exists" will break the
# --skip-existing functionality in twine
# ref: https://github.com/pypa/warehouse/issues/3482
# ref: https://github.com/pypa/twine/issues/332
"File already exists. See " +
request.help_url(_anchor="file-name-reuse"),
)
# Check to see if the file that was uploaded exists in our filename log
if request.db.query(
request.db.query(Filename).filter(
Filename.filename == filename).exists()).scalar():
raise _exc_with_message(
HTTPBadRequest,
"This filename has already been used, use a "
"different version. "
"See " + request.help_url(_anchor="file-name-reuse"),
)
# Check to see if uploading this file would create a duplicate sdist
# for the current release.
if (form.filetype.data == "sdist" and request.db.query(
request.db.query(File).filter((File.release == release) & (
File.packagetype == "sdist")).exists()).scalar()):
raise _exc_with_message(
HTTPBadRequest, "Only one sdist may be uploaded per release.")
# Check the file to make sure it is a valid distribution file.
if not _is_valid_dist_file(temporary_filename, form.filetype.data):
raise _exc_with_message(HTTPBadRequest,
"Invalid distribution file.")
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
for plat in plats:
if not _valid_platform_tag(plat):
raise _exc_with_message(
HTTPBadRequest,
"Binary wheel '{filename}' has an unsupported "
"platform tag '{plat}'.".format(filename=filename,
plat=plat),
)
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(
lambda: request.POST["gpg_signature"].file.read(8096),
b""):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(HTTPBadRequest,
"Signature too large.")
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(
HTTPBadRequest, "PGP signature isn't ASCII armored.")
else:
has_signature = False
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=file_hashes["md5"],
sha256_digest=file_hashes["sha256"],
blake2_256_digest=file_hashes["blake2_256"],
# Figure out what our filepath is going to be, we're going to use a
# directory structure based on the hash of the file contents. This
# will ensure that the contents of the file cannot change without
# it also changing the path that the file is saved too.
path="/".join([
file_hashes[PATH_HASHER][:2],
file_hashes[PATH_HASHER][2:4],
file_hashes[PATH_HASHER][4:],
filename,
]),
uploaded_via=request.user_agent,
)
request.db.add(file_)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="add {python_version} file {filename}".format(
python_version=file_.python_version,
filename=file_.filename),
submitted_by=request.user,
submitted_from=request.remote_addr,
))
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# committed.
storage = request.find_service(IFileStorage)
storage.store(
file_.path,
os.path.join(tmpdir, filename),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
if has_signature:
storage.store(
file_.pgp_path,
os.path.join(tmpdir, filename + ".asc"),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
# Log a successful upload
metrics.increment("warehouse.upload.ok",
tags=[f"filetype:{form.filetype.data}"])
return Response()
def file_upload(request):
# Before we do anything, if there isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(
HTTPForbidden,
"Invalid or non-existent authentication information.",
)
# distutils "helpfully" substitutes unknown, but "required" values with the
# string "UNKNOWN". This is basically never what anyone actually wants so
# we'll just go ahead and delete anything whose value is UNKNOWN.
for key in list(request.POST):
if request.POST.get(key) == "UNKNOWN":
del request.POST[key]
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Check if any fields were supplied as a tuple and have become a
# FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a
# FieldStorage, however.
# ref: https://github.com/pypa/warehouse/issues/2185
# ref: https://github.com/pypa/warehouse/issues/2491
for field in set(request.POST) - {'content', 'gpg_signature'}:
values = request.POST.getall(field)
if any(isinstance(value, FieldStorage) for value in values):
raise _exc_with_message(
HTTPBadRequest,
f"{field}: Should not be a tuple.",
)
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
form.classifiers.choices = [(c.classifier, c.classifier)
for c in all_classifiers]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
if field_name in form:
if form[field_name].description:
error_message = (
"{value!r} is an invalid value for {field}. ".format(
value=form[field_name].data,
field=form[field_name].description) +
"Error: {} ".format(form.errors[field_name][0]) + "see "
"https://packaging.python.org/specifications/core-metadata"
)
else:
error_message = "{field}: {msgs[0]}".format(
field=field_name,
msgs=form.errors[field_name],
)
else:
error_message = "Error: {}".format(form.errors[field_name][0])
raise _exc_with_message(
HTTPBadRequest,
error_message,
)
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(
HTTPBadRequest,
"Upload payload does not have a file.",
)
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
form.name.data)).one())
except NoResultFound:
# Check for AdminFlag set by a PyPI Administrator disabling new project
# registration, reasons for this include Spammers, security
# vulnerabilities, or just wanting to be lazy and not worry ;)
if AdminFlag.is_enabled(request.db,
'disallow-new-project-registration'):
raise _exc_with_message(
HTTPForbidden,
("New Project Registration Temporarily Disabled "
"See https://pypi.org/help#admin-intervention for details"),
) from None
# Ensure that user has at least one verified email address. This should
# reduce the ease of spam account creation and activity.
# TODO: Once legacy is shutdown consider the condition here, perhaps
# move to user.is_active or some other boolean
if not any(email.verified for email in request.user.emails):
raise _exc_with_message(
HTTPBadRequest,
("User {!r} has no verified email addresses, please verify "
"at least one address before registering a new project on "
"PyPI. See https://pypi.org/help/#verified-email "
"for more information.").format(request.user.username),
) from None
# Before we create the project, we're going to check our blacklist to
# see if this project is even allowed to be registered. If it is not,
# then we're going to deny the request to create this project.
if request.db.query(exists().where(
BlacklistedProject.name == func.normalize_pep426_name(
form.name.data))).scalar():
raise _exc_with_message(
HTTPBadRequest,
("The name {!r} is not allowed. "
"See https://pypi.org/help/#project-name "
"for more information.").format(form.name.data),
) from None
# Also check for collisions with Python Standard Library modules.
if (packaging.utils.canonicalize_name(form.name.data)
in STDLIB_PROHIBITTED):
raise _exc_with_message(
HTTPBadRequest,
("The name {!r} is not allowed (conflict with Python "
"Standard Library module name). See "
"https://pypi.org/help/#project-name for more information."
).format(form.name.data),
) from None
# The project doesn't exist in our database, so we'll add it along with
# a role setting the current user as the "Owner" of the project.
project = Project(name=form.name.data)
request.db.add(project)
request.db.add(
Role(user=request.user, project=project, role_name="Owner"))
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=project.name,
action="create",
submitted_by=request.user,
submitted_from=request.remote_addr,
), )
request.db.add(
JournalEntry(
name=project.name,
action="add Owner {}".format(request.user.username),
submitted_by=request.user,
submitted_from=request.remote_addr,
), )
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(
HTTPForbidden,
("The user '{0}' is not allowed to upload to project '{1}'. "
"See https://pypi.org/help#project-name for more information."
).format(request.user.username, project.name))
try:
canonical_version = packaging.utils.canonicalize_version(
form.version.data)
release = (request.db.query(Release).filter(
(Release.project == project)
& (Release.canonical_version == canonical_version)).one())
except MultipleResultsFound:
# There are multiple releases of this project which have the same
# canonical version that were uploaded before we checked for
# canonical version equivalence, so return the exact match instead
release = (request.db.query(
Release).filter((Release.project == project)
& (Release.version == form.version.data)).one())
except NoResultFound:
release = Release(
project=project,
_classifiers=[
c for c in all_classifiers
if c.classifier in form.classifiers.data
],
_pypi_hidden=False,
dependencies=list(
_construct_dependencies(
form, {
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
})),
canonical_version=canonical_version,
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary",
"description",
"license",
"author",
"author_email",
"maintainer",
"maintainer_email",
"keywords",
"platform",
"home_page",
"download_url",
"requires_python",
}
})
request.db.add(release)
# TODO: This should be handled by some sort of database trigger or
# a SQLAlchemy hook or the like instead of doing it inline in
# this view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="new release",
submitted_by=request.user,
submitted_from=request.remote_addr,
), )
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = (request.db.query(Release).filter(
Release.project == project).all())
for i, r in enumerate(
sorted(releases,
key=lambda x: packaging.version.parse(x.version))):
r._pypi_ordering = i
# TODO: Again, we should figure out a better solution to doing this than
# just inlining this inside this method.
if project.autohide:
for r in releases:
r._pypi_hidden = bool(not r == release)
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain any path separators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(
HTTPBadRequest,
"Cannot upload a file with '/' or '\\' in the name.",
)
# Make sure the filename ends with an allowed extension.
if _dist_file_regexes[project.allow_legacy_files].search(filename) is None:
raise _exc_with_message(
HTTPBadRequest,
"Invalid file extension. PEP 527 requires one of: .egg, .tar.gz, "
".whl, .zip (https://www.python.org/dev/peps/pep-0527/).")
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest,
"The filename for {!r} must start with {!r}.".format(
project.name,
prefix,
))
# Check the content type of what is being uploaded
if (not request.POST["content"].type
or request.POST["content"].type.startswith("image/")):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Ensure that the package filetpye is allowed.
# TODO: Once PEP 527 is completely implemented we should be able to delete
# this and just move it into the form itself.
if (not project.allow_legacy_files and form.filetype.data
not in {"sdist", "bdist_wheel", "bdist_egg"}):
raise _exc_with_message(HTTPBadRequest, "Unknown type of file.")
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
temporary_filename = os.path.join(tmpdir, filename)
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(temporary_filename, "wb") as fp:
file_size = 0
file_hashes = {
"md5": hashlib.md5(),
"sha256": hashlib.sha256(),
"blake2_256": hashlib.blake2b(digest_size=256 // 8),
}
for chunk in iter(lambda: request.POST["content"].file.read(8096),
b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(
HTTPBadRequest, "File too large. " +
"Limit for project {name!r} is {limit}MB".format(
name=project.name,
limit=file_size_limit // (1024 * 1024),
))
fp.write(chunk)
for hasher in file_hashes.values():
hasher.update(chunk)
# Take our hash functions and compute the final hashes for them now.
file_hashes = {
k: h.hexdigest().lower()
for k, h in file_hashes.items()
}
# Actually verify the digests that we've gotten. We're going to use
# hmac.compare_digest even though we probably don't actually need to
# because it's better safe than sorry. In the case of multiple digests
# we expect them all to be given.
if not all([
hmac.compare_digest(
getattr(form,
"{}_digest".format(digest_name)).data.lower(),
digest_value,
) for digest_name, digest_value in file_hashes.items()
if getattr(form, "{}_digest".format(digest_name)).data
]):
raise _exc_with_message(
HTTPBadRequest,
"The digest supplied does not match a digest calculated "
"from the uploaded file.")
# Check to see if the file that was uploaded exists already or not.
is_duplicate = _is_duplicate_file(request.db, filename, file_hashes)
if is_duplicate:
return Response()
elif is_duplicate is not None:
raise _exc_with_message(
HTTPBadRequest, "File already exists. "
"See " + request.route_url('help', _anchor='file-name-reuse'))
# Check to see if the file that was uploaded exists in our filename log
if (request.db.query(
request.db.query(Filename).filter(
Filename.filename == filename).exists()).scalar()):
raise _exc_with_message(
HTTPBadRequest,
"This filename has previously been used, you should use a "
"different version. "
"See " + request.route_url('help', _anchor='file-name-reuse'),
)
# Check to see if uploading this file would create a duplicate sdist
# for the current release.
if (form.filetype.data == "sdist" and request.db.query(
request.db.query(File).filter((File.release == release) & (
File.packagetype == "sdist")).exists()).scalar()):
raise _exc_with_message(
HTTPBadRequest,
"Only one sdist may be uploaded per release.",
)
# Check the file to make sure it is a valid distribution file.
if not _is_valid_dist_file(temporary_filename, form.filetype.data):
raise _exc_with_message(
HTTPBadRequest,
"Invalid distribution file.",
)
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
for plat in plats:
if not _valid_platform_tag(plat):
raise _exc_with_message(
HTTPBadRequest,
"Binary wheel '{filename}' has an unsupported "
"platform tag '{plat}'.".format(filename=filename,
plat=plat))
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(
lambda: request.POST["gpg_signature"].file.read(8096),
b""):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(
HTTPBadRequest,
"Signature too large.",
)
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(
HTTPBadRequest,
"PGP signature is not ASCII armored.",
)
else:
has_signature = False
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=file_hashes["md5"],
sha256_digest=file_hashes["sha256"],
blake2_256_digest=file_hashes["blake2_256"],
# Figure out what our filepath is going to be, we're going to use a
# directory structure based on the hash of the file contents. This
# will ensure that the contents of the file cannot change without
# it also changing the path that the file is saved too.
path="/".join([
file_hashes[PATH_HASHER][:2],
file_hashes[PATH_HASHER][2:4],
file_hashes[PATH_HASHER][4:],
filename,
]),
)
request.db.add(file_)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="add {python_version} file {filename}".format(
python_version=file_.python_version,
filename=file_.filename,
),
submitted_by=request.user,
submitted_from=request.remote_addr,
), )
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# committed.
storage = request.find_service(IFileStorage)
storage.store(
file_.path,
os.path.join(tmpdir, filename),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
if has_signature:
storage.store(
file_.pgp_path,
os.path.join(tmpdir, filename + ".asc"),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
# TODO: Once we no longer have the legacy code base running PyPI we can
# go ahead and delete this tiny bit of shim code, since it only
# exists to purge stuff on legacy PyPI when uploaded to Warehouse
old_domain = request.registry.settings.get("warehouse.legacy_domain")
if old_domain:
request.tm.get().addAfterCommitHook(
_legacy_purge,
args=["https://{}/pypi".format(old_domain)],
kws={"data": {
":action": "purge",
"project": project.name
}},
)
return Response()
def manage_project_roles(project, request, _form_class=CreateRoleForm):
user_service = request.find_service(IUserService, context=None)
form = _form_class(request.POST, user_service=user_service)
if request.method == "POST" and form.validate():
username = form.username.data
role_name = form.role_name.data
userid = user_service.find_userid(username)
user = user_service.get_user(userid)
if request.db.query(
request.db.query(Role).filter(
Role.user == user, Role.project == project,
Role.role_name == role_name).exists()).scalar():
request.session.flash(
f"User '{username}' already has {role_name} role for project",
queue="error",
)
elif user.primary_email is None or not user.primary_email.verified:
request.session.flash(
f"User '{username}' does not have a verified primary email "
f"address and cannot be added as a {role_name} for project",
queue="error",
)
else:
request.db.add(
Role(user=user, project=project,
role_name=form.role_name.data))
request.db.add(
JournalEntry(
name=project.name,
action=f"add {role_name} {username}",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
project.record_event(
tag="project:role:add",
ip_address=request.remote_addr,
additional={
"submitted_by": request.user.username,
"role_name": role_name,
"target_user": username,
},
)
owner_roles = (request.db.query(Role).join(Role.user).filter(
Role.role_name == "Owner", Role.project == project))
owner_users = {owner.user for owner in owner_roles}
# Don't send to the owner that added the new role
owner_users.discard(request.user)
# Don't send owners email to new user if they are now an owner
owner_users.discard(user)
send_collaborator_added_email(
request,
owner_users,
user=user,
submitter=request.user,
project_name=project.name,
role=form.role_name.data,
)
send_added_as_collaborator_email(
request,
user,
submitter=request.user,
project_name=project.name,
role=form.role_name.data,
)
request.session.flash(f"Added collaborator '{form.username.data}'",
queue="success")
form = _form_class(user_service=user_service)
roles = request.db.query(Role).join(User).filter(
Role.project == project).all()
# TODO: The following lines are a hack to handle multiple roles for a
# single user and should be removed when fixing GH-2745
roles_by_user = defaultdict(list)
for role in roles:
roles_by_user[role.user.username].append(role)
return {"project": project, "roles_by_user": roles_by_user, "form": form}
def change_project_role(project, request, _form_class=ChangeRoleForm):
# TODO: This view was modified to handle deleting multiple roles for a
# single user and should be updated when fixing GH-2745
form = _form_class(request.POST)
if form.validate():
role_ids = request.POST.getall("role_id")
if len(role_ids) > 1:
# This user has more than one role, so just delete all the ones
# that aren't what we want.
#
# TODO: This branch should be removed when fixing GH-2745.
roles = (request.db.query(Role).join(User).filter(
Role.id.in_(role_ids),
Role.project == project,
Role.role_name != form.role_name.data,
).all())
removing_self = any(
role.role_name == "Owner" and role.user == request.user
for role in roles)
if removing_self:
request.session.flash("Cannot remove yourself as Owner",
queue="error")
else:
for role in roles:
request.db.delete(role)
request.db.add(
JournalEntry(
name=project.name,
action=
f"remove {role.role_name} {role.user.username}",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
project.record_event(
tag="project:role:delete",
ip_address=request.remote_addr,
additional={
"submitted_by": request.user.username,
"role_name": role.role_name,
"target_user": role.user.username,
},
)
request.session.flash("Changed role", queue="success")
else:
# This user only has one role, so get it and change the type.
try:
role = (request.db.query(Role).join(User).filter(
Role.id == request.POST.get("role_id"),
Role.project == project).one())
if role.role_name == "Owner" and role.user == request.user:
request.session.flash("Cannot remove yourself as Owner",
queue="error")
else:
request.db.add(
JournalEntry(
name=project.name,
action="change {} {} to {}".format(
role.role_name, role.user.username,
form.role_name.data),
submitted_by=request.user,
submitted_from=request.remote_addr,
))
role.role_name = form.role_name.data
project.record_event(
tag="project:role:change",
ip_address=request.remote_addr,
additional={
"submitted_by": request.user.username,
"role_name": form.role_name.data,
"target_user": role.user.username,
},
)
request.session.flash("Changed role", queue="success")
except NoResultFound:
request.session.flash("Could not find role", queue="error")
return HTTPSeeOther(
request.route_path("manage.project.roles", project_name=project.name))
def delete_project_release(self):
if self.request.flags.enabled(AdminFlagValue.DISALLOW_DELETION):
self.request.session.flash(
("Project deletion temporarily disabled. "
"See https://pypi.org/help#admin-intervention for details."),
queue="error",
)
return HTTPSeeOther(
self.request.route_path(
"manage.project.release",
project_name=self.release.project.name,
version=self.release.version,
))
version = self.request.POST.get("confirm_version")
if not version:
self.request.session.flash("Confirm the request", queue="error")
return HTTPSeeOther(
self.request.route_path(
"manage.project.release",
project_name=self.release.project.name,
version=self.release.version,
))
if version != self.release.version:
self.request.session.flash(
"Could not delete release - " +
f"{version!r} is not the same as {self.release.version!r}",
queue="error",
)
return HTTPSeeOther(
self.request.route_path(
"manage.project.release",
project_name=self.release.project.name,
version=self.release.version,
))
self.request.db.add(
JournalEntry(
name=self.release.project.name,
action="remove release",
version=self.release.version,
submitted_by=self.request.user,
submitted_from=self.request.remote_addr,
))
self.release.project.record_event(
tag="project:release:remove",
ip_address=self.request.remote_addr,
additional={
"submitted_by": self.request.user.username,
"canonical_version": self.release.canonical_version,
},
)
self.request.db.delete(self.release)
self.request.session.flash(f"Deleted release {self.release.version!r}",
queue="success")
return HTTPSeeOther(
self.request.route_path("manage.project.releases",
project_name=self.release.project.name))
def add_blacklist(request):
project_name = request.POST.get("project")
if project_name is None:
raise HTTPBadRequest("Must have a project to confirm.")
comment = request.POST.get("comment", "")
# Verify that the user has confirmed the request to blacklist.
confirm = request.POST.get("confirm")
if not confirm:
request.session.flash(
"Must confirm the blacklist request.",
queue="error",
)
return HTTPSeeOther(request.current_route_path())
elif canonicalize_name(confirm) != canonicalize_name(project_name):
request.session.flash(
f"{confirm!r} is not the same as {project_name!r}",
queue="error",
)
return HTTPSeeOther(request.current_route_path())
# Add our requested blacklist.
request.db.add(
BlacklistedProject(
name=project_name,
comment=comment,
blacklisted_by=request.user,
))
# Go through and delete anything that we need to delete so that our
# blacklist actually blocks things and isn't ignored (since the blacklist
# only takes effect on new project registration).
# TODO: This should be in a generic function somewhere instead of putting
# it here, however this will have to do for now.
# TODO: We don't actually delete files from the data store. We should add
# some kind of garbage collection at some point.
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
project_name)).first())
if project is not None:
request.db.add(
JournalEntry(
name=project.name,
action="remove",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.db.query(Role).filter(Role.project == project).delete()
request.db.query(File).filter(File.name == project.name).delete()
(request.db.query(Dependency).filter(
Dependency.name == project.name).delete())
(request.db.execute(release_classifiers.delete().where(
release_classifiers.c.name == project.name)))
request.db.query(Release).filter(Release.name == project.name).delete()
request.db.query(Project).filter(Project.name == project.name).delete()
request.session.flash(
f"Successfully blacklisted {project_name!r}",
queue="success",
)
return HTTPSeeOther(request.route_path("admin.blacklist.list"))
def file_upload(request):
# Before we do anything, if there isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(
HTTPForbidden,
"Invalid or non-existent authentication information.",
)
# distutils "helpfully" substitutes unknown, but "required" values with the
# string "UNKNOWN". This is basically never what anyone actually wants so
# we'll just go ahead and delete anything whose value is UNKNOWN.
for key in list(request.POST):
if request.POST.get(key) == "UNKNOWN":
del request.POST[key]
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
form.classifiers.choices = [
(c.classifier, c.classifier) for c in all_classifiers
]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
raise _exc_with_message(
HTTPBadRequest,
"{field}: {msgs[0]}".format(
field=field_name,
msgs=form.errors[field_name],
),
)
# TODO: We need a better method of blocking names rather than just
# hardcoding some names into source control.
if form.name.data.lower() in {"requirements.txt", "rrequirements.txt"}:
raise _exc_with_message(
HTTPBadRequest,
"The name {!r} is not allowed.".format(form.name.data),
)
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(
HTTPBadRequest,
"Upload payload does not have a file.",
)
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (
request.db.query(Project)
.filter(
Project.normalized_name ==
func.normalize_pep426_name(form.name.data)).one()
)
except NoResultFound:
# The project doesn't exist in our database, so we'll add it along with
# a role setting the current user as the "Owner" of the project.
project = Project(name=form.name.data)
request.db.add(project)
request.db.add(
Role(user=request.user, project=project, role_name="Owner")
)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=project.name,
action="create",
submitted_by=request.user,
submitted_from=request.client_addr,
),
)
request.db.add(
JournalEntry(
name=project.name,
action="add Owner {}".format(request.user.username),
submitted_by=request.user,
submitted_from=request.client_addr,
),
)
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(
HTTPForbidden,
"You are not allowed to upload to {!r}.".format(project.name)
)
try:
release = (
request.db.query(Release)
.filter(
(Release.project == project) &
(Release.version == form.version.data)).one()
)
except NoResultFound:
release = Release(
project=project,
_classifiers=[
c for c in all_classifiers
if c.classifier in form.classifiers.data
],
_pypi_hidden=False,
dependencies=list(_construct_dependencies(
form,
{
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
}
)),
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary", "description", "license",
"author", "author_email", "maintainer", "maintainer_email",
"keywords", "platform",
"home_page", "download_url",
"requires_python",
}
}
)
request.db.add(release)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="new release",
submitted_by=request.user,
submitted_from=request.client_addr,
),
)
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = (
request.db.query(Release)
.filter(Release.project == project)
.all()
)
for i, r in enumerate(sorted(
releases, key=lambda x: packaging.version.parse(x.version))):
r._pypi_ordering = i
# TODO: Again, we should figure out a better solution to doing this than
# just inlining this inside this method.
if project.autohide:
for r in releases:
r._pypi_hidden = bool(not r == release)
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain any path separators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(
HTTPBadRequest,
"Cannot upload a file with '/' or '\\' in the name.",
)
# Make sure the filename ends with an allowed extension.
if _dist_file_re.search(filename) is None:
raise _exc_with_message(HTTPBadRequest, "Invalid file extension.")
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest,
"The filename for {!r} must start with {!r}.".format(
project.name,
prefix,
)
)
# Check the content type of what is being uploaded
if (not request.POST["content"].type or
request.POST["content"].type.startswith("image/")):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Check to see if the file that was uploaded exists already or not.
if request.db.query(
request.db.query(File)
.filter(File.filename == filename)
.exists()).scalar():
raise _exc_with_message(HTTPBadRequest, "File already exists.")
# Check to see if the file that was uploaded exists in our filename log.
if (request.db.query(
request.db.query(Filename)
.filter(Filename.filename == filename)
.exists()).scalar()):
raise _exc_with_message(
HTTPBadRequest,
"This filename has previously been used, you should use a "
"different version.",
)
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
temporary_filename = os.path.join(tmpdir, filename)
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(temporary_filename, "wb") as fp:
file_size = 0
file_hashes = {
"md5": hashlib.md5(),
"sha256": hashlib.sha256(),
"blake2_256": blake2b(digest_size=256 // 8),
}
for chunk in iter(
lambda: request.POST["content"].file.read(8096), b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(HTTPBadRequest, "File too large.")
fp.write(chunk)
for hasher in file_hashes.values():
hasher.update(chunk)
# Take our hash functions and compute the final hashes for them now.
file_hashes = {
k: h.hexdigest().lower()
for k, h in file_hashes.items()
}
# Actually verify the digests that we've gotten. We're going to use
# hmac.compare_digest even though we probably don't actually need to
# because it's better safe than sorry. In the case of multiple digests
# we expect them all to be given.
if not all([
hmac.compare_digest(
getattr(form, "{}_digest".format(digest_name)).data.lower(),
digest_value,
)
for digest_name, digest_value in file_hashes.items()
if getattr(form, "{}_digest".format(digest_name)).data
]):
raise _exc_with_message(
HTTPBadRequest,
"The digest supplied does not match a digest calculated "
"from the uploaded file."
)
# Check the file to make sure it is a valid distribution file.
if not _is_valid_dist_file(temporary_filename, form.filetype.data):
raise _exc_with_message(
HTTPBadRequest,
"Invalid distribution file.",
)
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
for plat in plats:
if not _valid_platform_tag(plat):
raise _exc_with_message(
HTTPBadRequest,
"Binary wheel '{filename}' has an unsupported "
"platform tag '{plat}'."
.format(filename=filename, plat=plat)
)
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(
lambda: request.POST["gpg_signature"].file.read(8096),
b""):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(
HTTPBadRequest,
"Signature too large.",
)
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(
HTTPBadRequest,
"PGP signature is not ASCII armored.",
)
else:
has_signature = False
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=file_hashes["md5"],
sha256_digest=file_hashes["sha256"],
blake2_256_digest=file_hashes["blake2_256"],
# Figure out what our filepath is going to be, we're going to use a
# directory structure based on the hash of the file contents. This
# will ensure that the contents of the file cannot change without
# it also changing the path that the file is saved too.
path="/".join([
file_hashes[PATH_HASHER][:2],
file_hashes[PATH_HASHER][2:4],
file_hashes[PATH_HASHER][4:],
filename,
]),
)
request.db.add(file_)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="add {python_version} file {filename}".format(
python_version=file_.python_version,
filename=file_.filename,
),
submitted_by=request.user,
submitted_from=request.client_addr,
),
)
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# committed.
storage = request.find_service(IFileStorage)
storage.store(
file_.path,
os.path.join(tmpdir, filename),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
if has_signature:
storage.store(
file_.pgp_path,
os.path.join(tmpdir, filename + ".asc"),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
# TODO: Once we no longer have the legacy code base running PyPI we can
# go ahead and delete this tiny bit of shim code, since it only
# exists to purge stuff on legacy PyPI when uploaded to Warehouse
old_domain = request.registry.settings.get("warehouse.legacy_domain")
if old_domain:
request.tm.get().addAfterCommitHook(
_legacy_purge,
args=["https://{}/pypi".format(old_domain)],
kws={"data": {":action": "purge", "project": project.name}},
)
return Response()
def verify_project_role(request):
token_service = request.find_service(ITokenService, name="email")
user_service = request.find_service(IUserService, context=None)
def _error(message):
request.session.flash(message, queue="error")
return HTTPSeeOther(request.route_path("manage.projects"))
try:
token = request.params.get("token")
data = token_service.loads(token)
except TokenExpired:
return _error(request._("Expired token: request a new project role invite"))
except TokenInvalid:
return _error(request._("Invalid token: request a new project role invite"))
except TokenMissing:
return _error(request._("Invalid token: no token supplied"))
# Check whether this token is being used correctly
if data.get("action") != "email-project-role-verify":
return _error(request._("Invalid token: not a collaboration invitation token"))
user = user_service.get_user(data.get("user_id"))
if user != request.user:
return _error(request._("Role invitation is not valid."))
project = (
request.db.query(Project).filter(Project.id == data.get("project_id")).one()
)
desired_role = data.get("desired_role")
role_invite = (
request.db.query(RoleInvitation)
.filter(RoleInvitation.project == project)
.filter(RoleInvitation.user == user)
.one_or_none()
)
if not role_invite:
return _error(request._("Role invitation no longer exists."))
# Use the renderer to bring up a confirmation page
# before adding as contributor
if request.method == "GET":
return {
"project_name": project.name,
"desired_role": desired_role,
}
elif request.method == "POST" and "decline" in request.POST:
request.db.delete(role_invite)
request.session.flash(
request._(
"Invitation for '${project_name}' is declined.",
mapping={"project_name": project.name},
),
queue="success",
)
return HTTPSeeOther(request.route_path("manage.projects"))
request.db.add(Role(user=user, project=project, role_name=desired_role))
request.db.delete(role_invite)
request.db.add(
JournalEntry(
name=project.name,
action=f"accepted {desired_role} {user.username}",
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
project.record_event(
tag="project:role:accepted",
ip_address=request.remote_addr,
additional={
"submitted_by": request.user.username,
"role_name": desired_role,
"target_user": user.username,
},
)
user.record_event(
tag="account:role:accepted",
ip_address=request.remote_addr,
additional={
"submitted_by": request.user.username,
"project_name": project.name,
"role_name": desired_role,
},
)
owner_roles = (
request.db.query(Role)
.filter(Role.project == project)
.filter(Role.role_name == "Owner")
.all()
)
owner_users = {owner.user for owner in owner_roles}
# Don't send email to new user if they are now an owner
owner_users.discard(user)
submitter_user = user_service.get_user(data.get("submitter_id"))
send_collaborator_added_email(
request,
owner_users,
user=user,
submitter=submitter_user,
project_name=project.name,
role=desired_role,
)
send_added_as_collaborator_email(
request,
user,
submitter=submitter_user,
project_name=project.name,
role=desired_role,
)
request.session.flash(
request._(
"You are now ${role} of the '${project_name}' project.",
mapping={"project_name": project.name, "role": desired_role},
),
queue="success",
)
if desired_role == "Owner":
return HTTPSeeOther(
request.route_path("manage.project.roles", project_name=project.name)
)
else:
return HTTPSeeOther(request.route_path("packaging.project", name=project.name))
