def user_delete(request): user = request.db.query(User).get(request.matchdict["user_id"]) if user.username != request.params.get("username"): request.session.flash("Wrong confirmation input", queue="error") return HTTPSeeOther(request.route_path("admin.user.detail", user_id=user.id)) # Delete all the user's projects projects = request.db.query(Project).filter( Project.name.in_( request.db.query(Project.name) .join(Role.project) .filter(Role.user == user) .subquery() ) ) for project in projects: request.db.add( JournalEntry( name=project.name, action="remove project", submitted_by=request.user, submitted_from=request.remote_addr, ) ) projects.delete(synchronize_session=False) # Update all journals to point to `deleted-user` instead deleted_user = request.db.query(User).filter(User.username == "deleted-user").one() journals = ( request.db.query(JournalEntry) .options(joinedload("submitted_by")) .filter(JournalEntry.submitted_by == user) .all() ) for journal in journals: journal.submitted_by = deleted_user # Delete the user request.db.delete(user) request.db.add( JournalEntry( name=f"user:{user.username}", action="nuke user", submitted_by=request.user, submitted_from=request.remote_addr, ) ) request.session.flash(f"Nuked user {user.username!r}", queue="success") return HTTPSeeOther(request.route_path("admin.user.list"))
def delete_project_role(project, request): # TODO: This view was modified to handle deleting multiple roles for a # single user and should be updated when fixing GH-2745 roles = (request.db.query(Role).filter( Role.id.in_(request.POST.getall("role_id")), Role.project == project).all()) removing_self = any(role.role_name == "Owner" and role.user == request.user for role in roles) if not roles: request.session.flash("Could not find role", queue="error") elif removing_self: request.session.flash("Cannot remove yourself as Owner", queue="error") else: for role in roles: request.db.delete(role) request.db.add( JournalEntry( name=project.name, action=f"remove {role.role_name} {role.user_name}", submitted_by=request.user, submitted_from=request.remote_addr, )) request.session.flash("Removed role", queue="success") return HTTPSeeOther( request.route_path("manage.project.roles", project_name=project.name))
def delete_role(project, request): confirm = request.POST.get("username") role_id = request.matchdict.get("role_id") role = request.db.query(Role).get(role_id) if not role: request.session.flash(f"This role no longer exists", queue="error") raise HTTPSeeOther( request.route_path("admin.project.detail", project_name=project.normalized_name)) if not confirm or confirm != role.user.username: request.session.flash("Confirm the request", queue="error") raise HTTPSeeOther( request.route_path("admin.project.detail", project_name=project.normalized_name)) request.session.flash( f"Removed '{role.user.username}' as '{role.role_name}' on '{project.name}'", queue="success", ) request.db.add( JournalEntry( name=project.name, action=f"remove {role.role_name} {role.user.username}", submitted_by=request.user, submitted_from=request.remote_addr, )) request.db.delete(role) return HTTPSeeOther( request.route_path("admin.project.detail", project_name=project.normalized_name))
def delete_project_release_file(self): def _error(message): self.request.session.flash(message, queue='error') return HTTPSeeOther( self.request.route_path( 'manage.project.release', project_name=self.release.project.name, version=self.release.version, ) ) project_name = self.request.POST.get('confirm_project_name') if not project_name: return _error("Must confirm the request") try: release_file = ( self.request.db.query(File) .filter( File.name == self.release.project.name, File.id == self.request.POST.get('file_id'), ) .one() ) except NoResultFound: return _error('Could not find file') if project_name != self.release.project.name: return _error( "Could not delete file - " + f"{project_name!r} is not the same as " f"{self.release.project.name!r}", ) self.request.db.add( JournalEntry( name=self.release.project.name, action=f"remove file {release_file.filename}", version=self.release.version, submitted_by=self.request.user, submitted_from=self.request.remote_addr, ), ) self.request.db.delete(release_file) self.request.session.flash( f"Successfully deleted file {release_file.filename!r}", queue="success", ) return HTTPSeeOther( self.request.route_path( 'manage.project.release', project_name=self.release.project.name, version=self.release.version, ) )
def delete_project_release_file(self): def _error(message): self.request.session.flash(message, queue="error") return HTTPSeeOther( self.request.route_path( "manage.project.release", project_name=self.release.project.name, version=self.release.version, )) project_name = self.request.POST.get("confirm_project_name") if not project_name: return _error("Confirm the request") try: release_file = (self.request.db.query(File).filter( File.release == self.release, File.id == self.request.POST.get("file_id"), ).one()) except NoResultFound: return _error("Could not find file") if project_name != self.release.project.name: return _error("Could not delete file - " + f"{project_name!r} is not the same as " f"{self.release.project.name!r}") self.request.db.add( JournalEntry( name=self.release.project.name, action=f"remove file {release_file.filename}", version=self.release.version, submitted_by=self.request.user, submitted_from=self.request.remote_addr, )) self.release.project.record_event( tag="project:release:file:remove", ip_address=self.request.remote_addr, additional={ "submitted_by": self.request.user.username, "canonical_version": self.release.canonical_version, "filename": release_file.filename, }, ) self.request.db.delete(release_file) self.request.session.flash(f"Deleted file {release_file.filename!r}", queue="success") return HTTPSeeOther( self.request.route_path( "manage.project.release", project_name=self.release.project.name, version=self.release.version, ))
def manage_project_roles(project, request, _form_class=CreateRoleForm): user_service = request.find_service(IUserService, context=None) form = _form_class(request.POST, user_service=user_service) if request.method == "POST" and form.validate(): username = form.username.data role_name = form.role_name.data userid = user_service.find_userid(username) user = user_service.get_user(userid) if (request.db.query( request.db.query(Role).filter( Role.user == user, Role.project == project, Role.role_name == role_name, ) .exists()).scalar()): request.session.flash( f"User '{username}' already has {role_name} role for project", queue="error" ) else: request.db.add( Role(user=user, project=project, role_name=form.role_name.data) ) request.db.add( JournalEntry( name=project.name, action=f"add {role_name} {username}", submitted_by=request.user, submitted_from=request.remote_addr, ), ) request.session.flash( f"Added collaborator '{form.username.data}'", queue="success" ) form = _form_class(user_service=user_service) roles = ( request.db.query(Role) .join(User) .filter(Role.project == project) .all() ) # TODO: The following lines are a hack to handle multiple roles for a # single user and should be removed when fixing GH-2745 roles_by_user = defaultdict(list) for role in roles: roles_by_user[role.user.username].append(role) return { "project": project, "roles_by_user": roles_by_user, "form": form, }
def delete_project_release(self): version = self.request.POST.get("confirm_version") if not version: self.request.session.flash("Confirm the request", queue="error") return HTTPSeeOther( self.request.route_path( "manage.project.release", project_name=self.release.project.name, version=self.release.version, ) ) if version != self.release.version: self.request.session.flash( "Could not delete release - " + f"{version!r} is not the same as {self.release.version!r}", queue="error", ) return HTTPSeeOther( self.request.route_path( "manage.project.release", project_name=self.release.project.name, version=self.release.version, ) ) self.request.db.add( JournalEntry( name=self.release.project.name, action="remove release", version=self.release.version, submitted_by=self.request.user, submitted_from=self.request.remote_addr, ) ) self.release.project.record_event( tag="project:release:remove", ip_address=self.request.remote_addr, additional={ "submitted_by": self.request.user.username, "canonical_version": self.release.canonical_version, }, ) self.request.db.delete(self.release) self.request.session.flash( f"Deleted release {self.release.version!r}", queue="success" ) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name ) )
def _nuke_user(user, request): # Delete all the user's projects projects = request.db.query(Project).filter( Project.name.in_( request.db.query(Project.name).join( Role.project).filter(Role.user == user).subquery())) for project in projects: request.db.add( JournalEntry( name=project.name, action="remove project", submitted_by=request.user, submitted_from=request.remote_addr, )) projects.delete(synchronize_session=False) # Update all journals to point to `deleted-user` instead deleted_user = request.db.query(User).filter( User.username == "deleted-user").one() journals = (request.db.query(JournalEntry).options( joinedload("submitted_by")).filter( JournalEntry.submitted_by == user).all()) for journal in journals: journal.submitted_by = deleted_user # Prohibit the username request.db.add( ProhibitedUserName(name=user.username.lower(), comment="nuked", prohibited_by=request.user)) # Delete the user request.db.delete(user) request.db.add( JournalEntry( name=f"user:{user.username}", action="nuke user", submitted_by=request.user, submitted_from=request.remote_addr, ))
def delete_project_release(self): version = self.request.POST.get('confirm_version') if not version: self.request.session.flash( "Must confirm the request", queue='error' ) return HTTPSeeOther( self.request.route_path( 'manage.project.release', project_name=self.release.project.name, version=self.release.version, ) ) if version != self.release.version: self.request.session.flash( "Could not delete release - " + f"{version!r} is not the same as {self.release.version!r}", queue="error", ) return HTTPSeeOther( self.request.route_path( 'manage.project.release', project_name=self.release.project.name, version=self.release.version, ) ) self.request.db.add( JournalEntry( name=self.release.project.name, action="remove", version=self.release.version, submitted_by=self.request.user, submitted_from=self.request.remote_addr, ), ) self.request.db.delete(self.release) self.request.session.flash( f"Successfully deleted release {self.release.version!r}", queue="success", ) return HTTPSeeOther( self.request.route_path( 'manage.project.releases', project_name=self.release.project.name, ) )
def add_role(project, request): username = request.POST.get("username") if not username: request.session.flash("Provide a username", queue="error") raise HTTPSeeOther( request.route_path("admin.project.detail", project_name=project.normalized_name)) try: user = request.db.query(User).filter(User.username == username).one() except NoResultFound: request.session.flash(f"Unknown username '{username}'", queue="error") raise HTTPSeeOther( request.route_path("admin.project.detail", project_name=project.normalized_name)) role_name = request.POST.get("role_name") if not role_name: request.session.flash("Provide a role", queue="error") raise HTTPSeeOther( request.route_path("admin.project.detail", project_name=project.normalized_name)) already_there = (request.db.query(Role).filter( Role.user == user, Role.project == project).count()) if already_there > 0: request.session.flash( f"User '{user.username}' already has a role on this project", queue="error") raise HTTPSeeOther( request.route_path("admin.project.detail", project_name=project.normalized_name)) request.db.add( JournalEntry( name=project.name, action=f"add {role_name} {user.username}", submitted_by=request.user, submitted_from=request.remote_addr, )) request.db.add(Role(role_name=role_name, user=user, project=project)) request.session.flash( f"Added '{user.username}' as '{role_name}' on '{project.name}'", queue="success") return HTTPSeeOther( request.route_path("admin.project.detail", project_name=project.normalized_name))
def destroy_docs(project, request, flash=True): request.db.add( JournalEntry( name=project.name, action="docdestroy", submitted_by=request.user, submitted_from=request.remote_addr, )) request.task(remove_documentation).delay(project.name) project.has_docs = False if flash: request.session.flash(f"Deleted docs for project {project.name!r}", queue="success")
def user_delete(request): user = request.db.query(User).get(request.matchdict['user_id']) if user.username != request.params.get('username'): print(user.username) print(request.params.get('username')) request.session.flash(f'Wrong confirmation input.', queue='error') return HTTPSeeOther( request.route_path('admin.user.detail', user_id=user.id) ) # Delete projects one by one so they are purged from the cache for project in user.projects: remove_project(project, request, flash=False) # Update all journals to point to `deleted-user` instead deleted_user = ( request.db.query(User) .filter(User.username == 'deleted-user') .one() ) journals = ( request.db.query(JournalEntry) .filter(JournalEntry.submitted_by == user) .all() ) for journal in journals: journal.submitted_by = deleted_user # Delete the user request.db.delete(user) request.db.add( JournalEntry( name=f'user:{user.username}', action=f'nuke user', submitted_by=request.user, submitted_from=request.remote_addr, ) ) request.session.flash(f'Nuked user {user.username!r}.', queue='success') return HTTPSeeOther(request.route_path('admin.user.list'))
def remove_project(project, request, flash=True): # TODO: We don't actually delete files from the data store. We should add # some kind of garbage collection at some point. request.db.add( JournalEntry( name=project.name, action="remove project", submitted_by=request.user, submitted_from=request.remote_addr, )) request.db.delete(project) # Flush so we can repeat this multiple times if necessary request.db.flush() if flash: request.session.flash(f"Deleted the project {project.name!r}", queue="success")
def remove_project(project, request, flash=True): # TODO: We don't actually delete files from the data store. We should add # some kind of garbage collection at some point. request.db.add( JournalEntry( name=project.name, action="remove", submitted_by=request.user, submitted_from=request.remote_addr, )) request.db.query(Role).filter(Role.project == project).delete() request.db.query(File).filter(File.name == project.name).delete() (request.db.query(Dependency).filter( Dependency.name == project.name).delete()) (request.db.execute(release_classifiers.delete().where( release_classifiers.c.name == project.name))) # Load the following objects into the session and individually delete them # so they are included in `session.deleted` and their cache keys are purged # Delete releases first, otherwise they will get cascade-deleted by the # project deletion and won't be purged for release in (request.db.query(Release).filter( Release.project == project).all()): request.db.delete(release) # Finally, delete the project request.db.delete(project) # Flush so we can repeat this multiple times if necessary request.db.flush() if flash: request.session.flash( f"Successfully deleted the project {project.name!r}.", queue="success", )
def user_delete(request): user = request.db.query(User).get(request.matchdict['user_id']) if user.username != request.params.get('username'): request.session.flash(f'Wrong confirmation input', queue='error') return HTTPSeeOther( request.route_path('admin.user.detail', user_id=user.id)) # Delete all the user's projects projects = (request.db.query(Project).filter( Project.name.in_( request.db.query(Project.name).join( Role.project).filter(Role.user == user).subquery()))) projects.delete(synchronize_session=False) # Update all journals to point to `deleted-user` instead deleted_user = (request.db.query(User).filter( User.username == 'deleted-user').one()) journals = (request.db.query(JournalEntry).filter( JournalEntry.submitted_by == user).all()) for journal in journals: journal.submitted_by = deleted_user # Delete the user request.db.delete(user) request.db.add( JournalEntry( name=f'user:{user.username}', action=f'nuke user', submitted_by=request.user, submitted_from=request.remote_addr, )) request.session.flash(f'Nuked user {user.username!r}', queue='success') return HTTPSeeOther(request.route_path('admin.user.list'))
def remove_project(project, request): # TODO: We don't actually delete files from the data store. We should add # some kind of garbage collection at some point. request.db.add( JournalEntry( name=project.name, action="remove", submitted_by=request.user, submitted_from=request.remote_addr, )) request.db.query(Role).filter(Role.project == project).delete() request.db.query(File).filter(File.name == project.name).delete() (request.db.query(Dependency).filter( Dependency.name == project.name).delete()) (request.db.execute(release_classifiers.delete().where( release_classifiers.c.name == project.name))) request.db.query(Release).filter(Release.name == project.name).delete() request.db.query(Project).filter(Project.name == project.name).delete() request.session.flash( f"Successfully deleted the project {project.name!r}.", queue="success", )
def file_upload(request): # If we're in read-only mode, let upload clients know if request.flags.enabled("read-only"): raise _exc_with_message( HTTPForbidden, "Read-only mode: Uploads are temporarily disabled") # Log an attempt to upload metrics = request.find_service(IMetricsService, context=None) metrics.increment("warehouse.upload.attempt") # Before we do anything, if there isn't an authenticated user with this # request, then we'll go ahead and bomb out. if request.authenticated_userid is None: raise _exc_with_message( HTTPForbidden, "Invalid or non-existent authentication information.") # Ensure that user has a verified, primary email address. This should both # reduce the ease of spam account creation and activity, as well as act as # a forcing function for https://github.com/pypa/warehouse/issues/3632. # TODO: Once https://github.com/pypa/warehouse/issues/3632 has been solved, # we might consider a different condition, possibly looking at # User.is_active instead. if not (request.user.primary_email and request.user.primary_email.verified): raise _exc_with_message( HTTPBadRequest, ("User {!r} does not have a verified primary email address. " "Please add a verified primary email before attempting to " "upload to PyPI. See {project_help} for more information." "for more information.").format( request.user.username, project_help=request.help_url(_anchor="verified-email"), ), ) from None # Do some cleanup of the various form fields for key in list(request.POST): value = request.POST.get(key) if isinstance(value, str): # distutils "helpfully" substitutes unknown, but "required" values # with the string "UNKNOWN". This is basically never what anyone # actually wants so we'll just go ahead and delete anything whose # value is UNKNOWN. if value.strip() == "UNKNOWN": del request.POST[key] # Escape NUL characters, which psycopg doesn't like if "\x00" in value: request.POST[key] = value.replace("\x00", "\\x00") # We require protocol_version 1, it's the only supported version however # passing a different version should raise an error. if request.POST.get("protocol_version", "1") != "1": raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.") # Check if any fields were supplied as a tuple and have become a # FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a # FieldStorage, however. # ref: https://github.com/pypa/warehouse/issues/2185 # ref: https://github.com/pypa/warehouse/issues/2491 for field in set(request.POST) - {"content", "gpg_signature"}: values = request.POST.getall(field) if any(isinstance(value, FieldStorage) for value in values): raise _exc_with_message(HTTPBadRequest, f"{field}: Should not be a tuple.") # Look up all of the valid classifiers all_classifiers = request.db.query(Classifier).all() # Validate and process the incoming metadata. form = MetadataForm(request.POST) # Add a validator for deprecated classifiers form.classifiers.validators.append(_no_deprecated_classifiers(request)) form.classifiers.choices = [(c.classifier, c.classifier) for c in all_classifiers] if not form.validate(): for field_name in _error_message_order: if field_name in form.errors: break else: field_name = sorted(form.errors.keys())[0] if field_name in form: field = form[field_name] if field.description and isinstance(field, wtforms.StringField): error_message = ( "{value!r} is an invalid value for {field}. ".format( value=field.data, field=field.description) + "Error: {} ".format(form.errors[field_name][0]) + "See " "https://packaging.python.org/specifications/core-metadata" ) else: error_message = "Invalid value for {field}. Error: {msgs[0]}".format( field=field_name, msgs=form.errors[field_name]) else: error_message = "Error: {}".format(form.errors[field_name][0]) raise _exc_with_message(HTTPBadRequest, error_message) # Ensure that we have file data in the request. if "content" not in request.POST: raise _exc_with_message(HTTPBadRequest, "Upload payload does not have a file.") # Look up the project first before doing anything else, this is so we can # automatically register it if we need to and can check permissions before # going any further. try: project = (request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name( form.name.data)).one()) except NoResultFound: # Check for AdminFlag set by a PyPI Administrator disabling new project # registration, reasons for this include Spammers, security # vulnerabilities, or just wanting to be lazy and not worry ;) if request.flags.enabled("disallow-new-project-registration"): raise _exc_with_message( HTTPForbidden, ("New project registration temporarily disabled. " "See {projecthelp} for details").format( projecthelp=request.help_url( _anchor="admin-intervention")), ) from None # Before we create the project, we're going to check our blacklist to # see if this project is even allowed to be registered. If it is not, # then we're going to deny the request to create this project. if request.db.query(exists().where( BlacklistedProject.name == func.normalize_pep426_name( form.name.data))).scalar(): raise _exc_with_message( HTTPBadRequest, ("The name {name!r} isn't allowed. " "See {projecthelp} " "for more information.").format( name=form.name.data, projecthelp=request.help_url(_anchor="project-name"), ), ) from None # Also check for collisions with Python Standard Library modules. if packaging.utils.canonicalize_name( form.name.data) in STDLIB_PROHIBITTED: raise _exc_with_message( HTTPBadRequest, ("The name {name!r} isn't allowed (conflict with Python " "Standard Library module name). See " "{projecthelp} for more information.").format( name=form.name.data, projecthelp=request.help_url(_anchor="project-name"), ), ) from None # The project doesn't exist in our database, so first we'll check for # projects with a similar name squattees = (request.db.query(Project).filter( func.levenshtein(Project.normalized_name, func.normalize_pep426_name(form.name.data)) <= 2). all()) # Next we'll create the project project = Project(name=form.name.data) request.db.add(project) # Now that the project exists, add any squats which it is the squatter for for squattee in squattees: request.db.add(Squat(squatter=project, squattee=squattee)) # Then we'll add a role setting the current user as the "Owner" of the # project. request.db.add( Role(user=request.user, project=project, role_name="Owner")) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=project.name, action="create", submitted_by=request.user, submitted_from=request.remote_addr, )) request.db.add( JournalEntry( name=project.name, action="add Owner {}".format(request.user.username), submitted_by=request.user, submitted_from=request.remote_addr, )) # Check that the user has permission to do things to this project, if this # is a new project this will act as a sanity check for the role we just # added above. if not request.has_permission("upload", project): raise _exc_with_message( HTTPForbidden, ("The credential associated with user '{0}' " "isn't allowed to upload to project '{1}'. " "See {2} for more information.").format( request.user.username, project.name, request.help_url(_anchor="project-name"), ), ) # Update name if it differs but is still equivalent. We don't need to check if # they are equivalent when normalized because that's already been done when we # queried for the project. if project.name != form.name.data: project.name = form.name.data # Render our description so we can save from having to render this data every time # we load a project description page. rendered = None if form.description.data: description_content_type = form.description_content_type.data if not description_content_type: description_content_type = "text/x-rst" rendered = readme.render(form.description.data, description_content_type, use_fallback=False) # Uploading should prevent broken rendered descriptions. if rendered is None: if form.description_content_type.data: message = ( "The description failed to render " "for '{description_content_type}'.").format( description_content_type=description_content_type) else: message = ("The description failed to render " "in the default format of reStructuredText.") raise _exc_with_message( HTTPBadRequest, "{message} See {projecthelp} for more information.".format( message=message, projecthelp=request.help_url( _anchor="description-content-type"), ), ) from None try: canonical_version = packaging.utils.canonicalize_version( form.version.data) release = (request.db.query(Release).filter( (Release.project == project) & (Release.canonical_version == canonical_version)).one()) except MultipleResultsFound: # There are multiple releases of this project which have the same # canonical version that were uploaded before we checked for # canonical version equivalence, so return the exact match instead release = (request.db.query( Release).filter((Release.project == project) & (Release.version == form.version.data)).one()) except NoResultFound: release = Release( project=project, _classifiers=[ c for c in all_classifiers if c.classifier in form.classifiers.data ], dependencies=list( _construct_dependencies( form, { "requires": DependencyKind.requires, "provides": DependencyKind.provides, "obsoletes": DependencyKind.obsoletes, "requires_dist": DependencyKind.requires_dist, "provides_dist": DependencyKind.provides_dist, "obsoletes_dist": DependencyKind.obsoletes_dist, "requires_external": DependencyKind.requires_external, "project_urls": DependencyKind.project_url, }, )), canonical_version=canonical_version, description=Description( content_type=form.description_content_type.data, raw=form.description.data or "", html=rendered or "", rendered_by=readme.renderer_version(), ), **{ k: getattr(form, k).data for k in { # This is a list of all the fields in the form that we # should pull off and insert into our new release. "version", "summary", "license", "author", "author_email", "maintainer", "maintainer_email", "keywords", "platform", "home_page", "download_url", "requires_python", } }, uploader=request.user, uploaded_via=request.user_agent, ) request.db.add(release) # TODO: This should be handled by some sort of database trigger or # a SQLAlchemy hook or the like instead of doing it inline in # this view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="new release", submitted_by=request.user, submitted_from=request.remote_addr, )) # TODO: We need a better solution to this than to just do it inline inside # this method. Ideally the version field would just be sortable, but # at least this should be some sort of hook or trigger. releases = (request.db.query(Release).filter( Release.project == project).options( orm.load_only(Release._pypi_ordering)).all()) for i, r in enumerate( sorted(releases, key=lambda x: packaging.version.parse(x.version))): r._pypi_ordering = i # Pull the filename out of our POST data. filename = request.POST["content"].filename # Make sure that the filename does not contain any path separators. if "/" in filename or "\\" in filename: raise _exc_with_message( HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name.") # Make sure the filename ends with an allowed extension. if _dist_file_regexes[project.allow_legacy_files].search(filename) is None: raise _exc_with_message( HTTPBadRequest, "Invalid file extension: Use .egg, .tar.gz, .whl or .zip " "extension. (https://www.python.org/dev/peps/pep-0527)", ) # Make sure that our filename matches the project that it is being uploaded # to. prefix = pkg_resources.safe_name(project.name).lower() if not pkg_resources.safe_name(filename).lower().startswith(prefix): raise _exc_with_message( HTTPBadRequest, "Start filename for {!r} with {!r}.".format(project.name, prefix), ) # Check the content type of what is being uploaded if not request.POST["content"].type or request.POST[ "content"].type.startswith("image/"): raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.") # Ensure that the package filetype is allowed. # TODO: Once PEP 527 is completely implemented we should be able to delete # this and just move it into the form itself. if not project.allow_legacy_files and form.filetype.data not in { "sdist", "bdist_wheel", "bdist_egg", }: raise _exc_with_message(HTTPBadRequest, "Unknown type of file.") # The project may or may not have a file size specified on the project, if # it does then it may or may not be smaller or larger than our global file # size limits. file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit])) with tempfile.TemporaryDirectory() as tmpdir: temporary_filename = os.path.join(tmpdir, filename) # Buffer the entire file onto disk, checking the hash of the file as we # go along. with open(temporary_filename, "wb") as fp: file_size = 0 file_hashes = { "md5": hashlib.md5(), "sha256": hashlib.sha256(), "blake2_256": hashlib.blake2b(digest_size=256 // 8), } for chunk in iter(lambda: request.POST["content"].file.read(8096), b""): file_size += len(chunk) if file_size > file_size_limit: raise _exc_with_message( HTTPBadRequest, "File too large. " + "Limit for project {name!r} is {limit} MB. ".format( name=project.name, limit=file_size_limit // (1024 * 1024)) + "See " + request.help_url(_anchor="file-size-limit"), ) fp.write(chunk) for hasher in file_hashes.values(): hasher.update(chunk) # Take our hash functions and compute the final hashes for them now. file_hashes = { k: h.hexdigest().lower() for k, h in file_hashes.items() } # Actually verify the digests that we've gotten. We're going to use # hmac.compare_digest even though we probably don't actually need to # because it's better safe than sorry. In the case of multiple digests # we expect them all to be given. if not all([ hmac.compare_digest( getattr(form, "{}_digest".format(digest_name)).data.lower(), digest_value, ) for digest_name, digest_value in file_hashes.items() if getattr(form, "{}_digest".format(digest_name)).data ]): raise _exc_with_message( HTTPBadRequest, "The digest supplied does not match a digest calculated " "from the uploaded file.", ) # Check to see if the file that was uploaded exists already or not. is_duplicate = _is_duplicate_file(request.db, filename, file_hashes) if is_duplicate: return Response() elif is_duplicate is not None: raise _exc_with_message( HTTPBadRequest, # Note: Changing this error message to something that doesn't # start with "File already exists" will break the # --skip-existing functionality in twine # ref: https://github.com/pypa/warehouse/issues/3482 # ref: https://github.com/pypa/twine/issues/332 "File already exists. See " + request.help_url(_anchor="file-name-reuse"), ) # Check to see if the file that was uploaded exists in our filename log if request.db.query( request.db.query(Filename).filter( Filename.filename == filename).exists()).scalar(): raise _exc_with_message( HTTPBadRequest, "This filename has already been used, use a " "different version. " "See " + request.help_url(_anchor="file-name-reuse"), ) # Check to see if uploading this file would create a duplicate sdist # for the current release. if (form.filetype.data == "sdist" and request.db.query( request.db.query(File).filter((File.release == release) & ( File.packagetype == "sdist")).exists()).scalar()): raise _exc_with_message( HTTPBadRequest, "Only one sdist may be uploaded per release.") # Check the file to make sure it is a valid distribution file. if not _is_valid_dist_file(temporary_filename, form.filetype.data): raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.") # Check that if it's a binary wheel, it's on a supported platform if filename.endswith(".whl"): wheel_info = _wheel_file_re.match(filename) plats = wheel_info.group("plat").split(".") for plat in plats: if not _valid_platform_tag(plat): raise _exc_with_message( HTTPBadRequest, "Binary wheel '{filename}' has an unsupported " "platform tag '{plat}'.".format(filename=filename, plat=plat), ) # Also buffer the entire signature file to disk. if "gpg_signature" in request.POST: has_signature = True with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp: signature_size = 0 for chunk in iter( lambda: request.POST["gpg_signature"].file.read(8096), b""): signature_size += len(chunk) if signature_size > MAX_SIGSIZE: raise _exc_with_message(HTTPBadRequest, "Signature too large.") fp.write(chunk) # Check whether signature is ASCII armored with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp: if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"): raise _exc_with_message( HTTPBadRequest, "PGP signature isn't ASCII armored.") else: has_signature = False # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add(Filename(filename=filename)) # Store the information about the file in the database. file_ = File( release=release, filename=filename, python_version=form.pyversion.data, packagetype=form.filetype.data, comment_text=form.comment.data, size=file_size, has_signature=bool(has_signature), md5_digest=file_hashes["md5"], sha256_digest=file_hashes["sha256"], blake2_256_digest=file_hashes["blake2_256"], # Figure out what our filepath is going to be, we're going to use a # directory structure based on the hash of the file contents. This # will ensure that the contents of the file cannot change without # it also changing the path that the file is saved too. path="/".join([ file_hashes[PATH_HASHER][:2], file_hashes[PATH_HASHER][2:4], file_hashes[PATH_HASHER][4:], filename, ]), uploaded_via=request.user_agent, ) request.db.add(file_) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="add {python_version} file {filename}".format( python_version=file_.python_version, filename=file_.filename), submitted_by=request.user, submitted_from=request.remote_addr, )) # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is # committed. storage = request.find_service(IFileStorage) storage.store( file_.path, os.path.join(tmpdir, filename), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) if has_signature: storage.store( file_.pgp_path, os.path.join(tmpdir, filename + ".asc"), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) # Log a successful upload metrics.increment("warehouse.upload.ok", tags=[f"filetype:{form.filetype.data}"]) return Response()
def file_upload(request): # Before we do anything, if there isn't an authenticated user with this # request, then we'll go ahead and bomb out. if request.authenticated_userid is None: raise _exc_with_message( HTTPForbidden, "Invalid or non-existent authentication information.", ) # distutils "helpfully" substitutes unknown, but "required" values with the # string "UNKNOWN". This is basically never what anyone actually wants so # we'll just go ahead and delete anything whose value is UNKNOWN. for key in list(request.POST): if request.POST.get(key) == "UNKNOWN": del request.POST[key] # We require protocol_version 1, it's the only supported version however # passing a different version should raise an error. if request.POST.get("protocol_version", "1") != "1": raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.") # Check if any fields were supplied as a tuple and have become a # FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a # FieldStorage, however. # ref: https://github.com/pypa/warehouse/issues/2185 # ref: https://github.com/pypa/warehouse/issues/2491 for field in set(request.POST) - {'content', 'gpg_signature'}: values = request.POST.getall(field) if any(isinstance(value, FieldStorage) for value in values): raise _exc_with_message( HTTPBadRequest, f"{field}: Should not be a tuple.", ) # Look up all of the valid classifiers all_classifiers = request.db.query(Classifier).all() # Validate and process the incoming metadata. form = MetadataForm(request.POST) form.classifiers.choices = [(c.classifier, c.classifier) for c in all_classifiers] if not form.validate(): for field_name in _error_message_order: if field_name in form.errors: break else: field_name = sorted(form.errors.keys())[0] if field_name in form: if form[field_name].description: error_message = ( "{value!r} is an invalid value for {field}. ".format( value=form[field_name].data, field=form[field_name].description) + "Error: {} ".format(form.errors[field_name][0]) + "see " "https://packaging.python.org/specifications/core-metadata" ) else: error_message = "{field}: {msgs[0]}".format( field=field_name, msgs=form.errors[field_name], ) else: error_message = "Error: {}".format(form.errors[field_name][0]) raise _exc_with_message( HTTPBadRequest, error_message, ) # Ensure that we have file data in the request. if "content" not in request.POST: raise _exc_with_message( HTTPBadRequest, "Upload payload does not have a file.", ) # Look up the project first before doing anything else, this is so we can # automatically register it if we need to and can check permissions before # going any further. try: project = (request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name( form.name.data)).one()) except NoResultFound: # Check for AdminFlag set by a PyPI Administrator disabling new project # registration, reasons for this include Spammers, security # vulnerabilities, or just wanting to be lazy and not worry ;) if AdminFlag.is_enabled(request.db, 'disallow-new-project-registration'): raise _exc_with_message( HTTPForbidden, ("New Project Registration Temporarily Disabled " "See https://pypi.org/help#admin-intervention for details"), ) from None # Ensure that user has at least one verified email address. This should # reduce the ease of spam account creation and activity. # TODO: Once legacy is shutdown consider the condition here, perhaps # move to user.is_active or some other boolean if not any(email.verified for email in request.user.emails): raise _exc_with_message( HTTPBadRequest, ("User {!r} has no verified email addresses, please verify " "at least one address before registering a new project on " "PyPI. See https://pypi.org/help/#verified-email " "for more information.").format(request.user.username), ) from None # Before we create the project, we're going to check our blacklist to # see if this project is even allowed to be registered. If it is not, # then we're going to deny the request to create this project. if request.db.query(exists().where( BlacklistedProject.name == func.normalize_pep426_name( form.name.data))).scalar(): raise _exc_with_message( HTTPBadRequest, ("The name {!r} is not allowed. " "See https://pypi.org/help/#project-name " "for more information.").format(form.name.data), ) from None # Also check for collisions with Python Standard Library modules. if (packaging.utils.canonicalize_name(form.name.data) in STDLIB_PROHIBITTED): raise _exc_with_message( HTTPBadRequest, ("The name {!r} is not allowed (conflict with Python " "Standard Library module name). See " "https://pypi.org/help/#project-name for more information." ).format(form.name.data), ) from None # The project doesn't exist in our database, so we'll add it along with # a role setting the current user as the "Owner" of the project. project = Project(name=form.name.data) request.db.add(project) request.db.add( Role(user=request.user, project=project, role_name="Owner")) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=project.name, action="create", submitted_by=request.user, submitted_from=request.remote_addr, ), ) request.db.add( JournalEntry( name=project.name, action="add Owner {}".format(request.user.username), submitted_by=request.user, submitted_from=request.remote_addr, ), ) # Check that the user has permission to do things to this project, if this # is a new project this will act as a sanity check for the role we just # added above. if not request.has_permission("upload", project): raise _exc_with_message( HTTPForbidden, ("The user '{0}' is not allowed to upload to project '{1}'. " "See https://pypi.org/help#project-name for more information." ).format(request.user.username, project.name)) try: canonical_version = packaging.utils.canonicalize_version( form.version.data) release = (request.db.query(Release).filter( (Release.project == project) & (Release.canonical_version == canonical_version)).one()) except MultipleResultsFound: # There are multiple releases of this project which have the same # canonical version that were uploaded before we checked for # canonical version equivalence, so return the exact match instead release = (request.db.query( Release).filter((Release.project == project) & (Release.version == form.version.data)).one()) except NoResultFound: release = Release( project=project, _classifiers=[ c for c in all_classifiers if c.classifier in form.classifiers.data ], _pypi_hidden=False, dependencies=list( _construct_dependencies( form, { "requires": DependencyKind.requires, "provides": DependencyKind.provides, "obsoletes": DependencyKind.obsoletes, "requires_dist": DependencyKind.requires_dist, "provides_dist": DependencyKind.provides_dist, "obsoletes_dist": DependencyKind.obsoletes_dist, "requires_external": DependencyKind.requires_external, "project_urls": DependencyKind.project_url, })), canonical_version=canonical_version, **{ k: getattr(form, k).data for k in { # This is a list of all the fields in the form that we # should pull off and insert into our new release. "version", "summary", "description", "license", "author", "author_email", "maintainer", "maintainer_email", "keywords", "platform", "home_page", "download_url", "requires_python", } }) request.db.add(release) # TODO: This should be handled by some sort of database trigger or # a SQLAlchemy hook or the like instead of doing it inline in # this view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="new release", submitted_by=request.user, submitted_from=request.remote_addr, ), ) # TODO: We need a better solution to this than to just do it inline inside # this method. Ideally the version field would just be sortable, but # at least this should be some sort of hook or trigger. releases = (request.db.query(Release).filter( Release.project == project).all()) for i, r in enumerate( sorted(releases, key=lambda x: packaging.version.parse(x.version))): r._pypi_ordering = i # TODO: Again, we should figure out a better solution to doing this than # just inlining this inside this method. if project.autohide: for r in releases: r._pypi_hidden = bool(not r == release) # Pull the filename out of our POST data. filename = request.POST["content"].filename # Make sure that the filename does not contain any path separators. if "/" in filename or "\\" in filename: raise _exc_with_message( HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name.", ) # Make sure the filename ends with an allowed extension. if _dist_file_regexes[project.allow_legacy_files].search(filename) is None: raise _exc_with_message( HTTPBadRequest, "Invalid file extension. PEP 527 requires one of: .egg, .tar.gz, " ".whl, .zip (https://www.python.org/dev/peps/pep-0527/).") # Make sure that our filename matches the project that it is being uploaded # to. prefix = pkg_resources.safe_name(project.name).lower() if not pkg_resources.safe_name(filename).lower().startswith(prefix): raise _exc_with_message( HTTPBadRequest, "The filename for {!r} must start with {!r}.".format( project.name, prefix, )) # Check the content type of what is being uploaded if (not request.POST["content"].type or request.POST["content"].type.startswith("image/")): raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.") # Ensure that the package filetpye is allowed. # TODO: Once PEP 527 is completely implemented we should be able to delete # this and just move it into the form itself. if (not project.allow_legacy_files and form.filetype.data not in {"sdist", "bdist_wheel", "bdist_egg"}): raise _exc_with_message(HTTPBadRequest, "Unknown type of file.") # The project may or may not have a file size specified on the project, if # it does then it may or may not be smaller or larger than our global file # size limits. file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit])) with tempfile.TemporaryDirectory() as tmpdir: temporary_filename = os.path.join(tmpdir, filename) # Buffer the entire file onto disk, checking the hash of the file as we # go along. with open(temporary_filename, "wb") as fp: file_size = 0 file_hashes = { "md5": hashlib.md5(), "sha256": hashlib.sha256(), "blake2_256": hashlib.blake2b(digest_size=256 // 8), } for chunk in iter(lambda: request.POST["content"].file.read(8096), b""): file_size += len(chunk) if file_size > file_size_limit: raise _exc_with_message( HTTPBadRequest, "File too large. " + "Limit for project {name!r} is {limit}MB".format( name=project.name, limit=file_size_limit // (1024 * 1024), )) fp.write(chunk) for hasher in file_hashes.values(): hasher.update(chunk) # Take our hash functions and compute the final hashes for them now. file_hashes = { k: h.hexdigest().lower() for k, h in file_hashes.items() } # Actually verify the digests that we've gotten. We're going to use # hmac.compare_digest even though we probably don't actually need to # because it's better safe than sorry. In the case of multiple digests # we expect them all to be given. if not all([ hmac.compare_digest( getattr(form, "{}_digest".format(digest_name)).data.lower(), digest_value, ) for digest_name, digest_value in file_hashes.items() if getattr(form, "{}_digest".format(digest_name)).data ]): raise _exc_with_message( HTTPBadRequest, "The digest supplied does not match a digest calculated " "from the uploaded file.") # Check to see if the file that was uploaded exists already or not. is_duplicate = _is_duplicate_file(request.db, filename, file_hashes) if is_duplicate: return Response() elif is_duplicate is not None: raise _exc_with_message( HTTPBadRequest, "File already exists. " "See " + request.route_url('help', _anchor='file-name-reuse')) # Check to see if the file that was uploaded exists in our filename log if (request.db.query( request.db.query(Filename).filter( Filename.filename == filename).exists()).scalar()): raise _exc_with_message( HTTPBadRequest, "This filename has previously been used, you should use a " "different version. " "See " + request.route_url('help', _anchor='file-name-reuse'), ) # Check to see if uploading this file would create a duplicate sdist # for the current release. if (form.filetype.data == "sdist" and request.db.query( request.db.query(File).filter((File.release == release) & ( File.packagetype == "sdist")).exists()).scalar()): raise _exc_with_message( HTTPBadRequest, "Only one sdist may be uploaded per release.", ) # Check the file to make sure it is a valid distribution file. if not _is_valid_dist_file(temporary_filename, form.filetype.data): raise _exc_with_message( HTTPBadRequest, "Invalid distribution file.", ) # Check that if it's a binary wheel, it's on a supported platform if filename.endswith(".whl"): wheel_info = _wheel_file_re.match(filename) plats = wheel_info.group("plat").split(".") for plat in plats: if not _valid_platform_tag(plat): raise _exc_with_message( HTTPBadRequest, "Binary wheel '{filename}' has an unsupported " "platform tag '{plat}'.".format(filename=filename, plat=plat)) # Also buffer the entire signature file to disk. if "gpg_signature" in request.POST: has_signature = True with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp: signature_size = 0 for chunk in iter( lambda: request.POST["gpg_signature"].file.read(8096), b""): signature_size += len(chunk) if signature_size > MAX_SIGSIZE: raise _exc_with_message( HTTPBadRequest, "Signature too large.", ) fp.write(chunk) # Check whether signature is ASCII armored with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp: if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"): raise _exc_with_message( HTTPBadRequest, "PGP signature is not ASCII armored.", ) else: has_signature = False # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add(Filename(filename=filename)) # Store the information about the file in the database. file_ = File( release=release, filename=filename, python_version=form.pyversion.data, packagetype=form.filetype.data, comment_text=form.comment.data, size=file_size, has_signature=bool(has_signature), md5_digest=file_hashes["md5"], sha256_digest=file_hashes["sha256"], blake2_256_digest=file_hashes["blake2_256"], # Figure out what our filepath is going to be, we're going to use a # directory structure based on the hash of the file contents. This # will ensure that the contents of the file cannot change without # it also changing the path that the file is saved too. path="/".join([ file_hashes[PATH_HASHER][:2], file_hashes[PATH_HASHER][2:4], file_hashes[PATH_HASHER][4:], filename, ]), ) request.db.add(file_) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="add {python_version} file {filename}".format( python_version=file_.python_version, filename=file_.filename, ), submitted_by=request.user, submitted_from=request.remote_addr, ), ) # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is # committed. storage = request.find_service(IFileStorage) storage.store( file_.path, os.path.join(tmpdir, filename), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) if has_signature: storage.store( file_.pgp_path, os.path.join(tmpdir, filename + ".asc"), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) # TODO: Once we no longer have the legacy code base running PyPI we can # go ahead and delete this tiny bit of shim code, since it only # exists to purge stuff on legacy PyPI when uploaded to Warehouse old_domain = request.registry.settings.get("warehouse.legacy_domain") if old_domain: request.tm.get().addAfterCommitHook( _legacy_purge, args=["https://{}/pypi".format(old_domain)], kws={"data": { ":action": "purge", "project": project.name }}, ) return Response()
def manage_project_roles(project, request, _form_class=CreateRoleForm): user_service = request.find_service(IUserService, context=None) form = _form_class(request.POST, user_service=user_service) if request.method == "POST" and form.validate(): username = form.username.data role_name = form.role_name.data userid = user_service.find_userid(username) user = user_service.get_user(userid) if request.db.query( request.db.query(Role).filter( Role.user == user, Role.project == project, Role.role_name == role_name).exists()).scalar(): request.session.flash( f"User '{username}' already has {role_name} role for project", queue="error", ) elif user.primary_email is None or not user.primary_email.verified: request.session.flash( f"User '{username}' does not have a verified primary email " f"address and cannot be added as a {role_name} for project", queue="error", ) else: request.db.add( Role(user=user, project=project, role_name=form.role_name.data)) request.db.add( JournalEntry( name=project.name, action=f"add {role_name} {username}", submitted_by=request.user, submitted_from=request.remote_addr, )) project.record_event( tag="project:role:add", ip_address=request.remote_addr, additional={ "submitted_by": request.user.username, "role_name": role_name, "target_user": username, }, ) owner_roles = (request.db.query(Role).join(Role.user).filter( Role.role_name == "Owner", Role.project == project)) owner_users = {owner.user for owner in owner_roles} # Don't send to the owner that added the new role owner_users.discard(request.user) # Don't send owners email to new user if they are now an owner owner_users.discard(user) send_collaborator_added_email( request, owner_users, user=user, submitter=request.user, project_name=project.name, role=form.role_name.data, ) send_added_as_collaborator_email( request, user, submitter=request.user, project_name=project.name, role=form.role_name.data, ) request.session.flash(f"Added collaborator '{form.username.data}'", queue="success") form = _form_class(user_service=user_service) roles = request.db.query(Role).join(User).filter( Role.project == project).all() # TODO: The following lines are a hack to handle multiple roles for a # single user and should be removed when fixing GH-2745 roles_by_user = defaultdict(list) for role in roles: roles_by_user[role.user.username].append(role) return {"project": project, "roles_by_user": roles_by_user, "form": form}
def change_project_role(project, request, _form_class=ChangeRoleForm): # TODO: This view was modified to handle deleting multiple roles for a # single user and should be updated when fixing GH-2745 form = _form_class(request.POST) if form.validate(): role_ids = request.POST.getall("role_id") if len(role_ids) > 1: # This user has more than one role, so just delete all the ones # that aren't what we want. # # TODO: This branch should be removed when fixing GH-2745. roles = (request.db.query(Role).join(User).filter( Role.id.in_(role_ids), Role.project == project, Role.role_name != form.role_name.data, ).all()) removing_self = any( role.role_name == "Owner" and role.user == request.user for role in roles) if removing_self: request.session.flash("Cannot remove yourself as Owner", queue="error") else: for role in roles: request.db.delete(role) request.db.add( JournalEntry( name=project.name, action= f"remove {role.role_name} {role.user.username}", submitted_by=request.user, submitted_from=request.remote_addr, )) project.record_event( tag="project:role:delete", ip_address=request.remote_addr, additional={ "submitted_by": request.user.username, "role_name": role.role_name, "target_user": role.user.username, }, ) request.session.flash("Changed role", queue="success") else: # This user only has one role, so get it and change the type. try: role = (request.db.query(Role).join(User).filter( Role.id == request.POST.get("role_id"), Role.project == project).one()) if role.role_name == "Owner" and role.user == request.user: request.session.flash("Cannot remove yourself as Owner", queue="error") else: request.db.add( JournalEntry( name=project.name, action="change {} {} to {}".format( role.role_name, role.user.username, form.role_name.data), submitted_by=request.user, submitted_from=request.remote_addr, )) role.role_name = form.role_name.data project.record_event( tag="project:role:change", ip_address=request.remote_addr, additional={ "submitted_by": request.user.username, "role_name": form.role_name.data, "target_user": role.user.username, }, ) request.session.flash("Changed role", queue="success") except NoResultFound: request.session.flash("Could not find role", queue="error") return HTTPSeeOther( request.route_path("manage.project.roles", project_name=project.name))
def delete_project_release(self): if self.request.flags.enabled(AdminFlagValue.DISALLOW_DELETION): self.request.session.flash( ("Project deletion temporarily disabled. " "See https://pypi.org/help#admin-intervention for details."), queue="error", ) return HTTPSeeOther( self.request.route_path( "manage.project.release", project_name=self.release.project.name, version=self.release.version, )) version = self.request.POST.get("confirm_version") if not version: self.request.session.flash("Confirm the request", queue="error") return HTTPSeeOther( self.request.route_path( "manage.project.release", project_name=self.release.project.name, version=self.release.version, )) if version != self.release.version: self.request.session.flash( "Could not delete release - " + f"{version!r} is not the same as {self.release.version!r}", queue="error", ) return HTTPSeeOther( self.request.route_path( "manage.project.release", project_name=self.release.project.name, version=self.release.version, )) self.request.db.add( JournalEntry( name=self.release.project.name, action="remove release", version=self.release.version, submitted_by=self.request.user, submitted_from=self.request.remote_addr, )) self.release.project.record_event( tag="project:release:remove", ip_address=self.request.remote_addr, additional={ "submitted_by": self.request.user.username, "canonical_version": self.release.canonical_version, }, ) self.request.db.delete(self.release) self.request.session.flash(f"Deleted release {self.release.version!r}", queue="success") return HTTPSeeOther( self.request.route_path("manage.project.releases", project_name=self.release.project.name))
def add_blacklist(request): project_name = request.POST.get("project") if project_name is None: raise HTTPBadRequest("Must have a project to confirm.") comment = request.POST.get("comment", "") # Verify that the user has confirmed the request to blacklist. confirm = request.POST.get("confirm") if not confirm: request.session.flash( "Must confirm the blacklist request.", queue="error", ) return HTTPSeeOther(request.current_route_path()) elif canonicalize_name(confirm) != canonicalize_name(project_name): request.session.flash( f"{confirm!r} is not the same as {project_name!r}", queue="error", ) return HTTPSeeOther(request.current_route_path()) # Add our requested blacklist. request.db.add( BlacklistedProject( name=project_name, comment=comment, blacklisted_by=request.user, )) # Go through and delete anything that we need to delete so that our # blacklist actually blocks things and isn't ignored (since the blacklist # only takes effect on new project registration). # TODO: This should be in a generic function somewhere instead of putting # it here, however this will have to do for now. # TODO: We don't actually delete files from the data store. We should add # some kind of garbage collection at some point. project = (request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name( project_name)).first()) if project is not None: request.db.add( JournalEntry( name=project.name, action="remove", submitted_by=request.user, submitted_from=request.remote_addr, )) request.db.query(Role).filter(Role.project == project).delete() request.db.query(File).filter(File.name == project.name).delete() (request.db.query(Dependency).filter( Dependency.name == project.name).delete()) (request.db.execute(release_classifiers.delete().where( release_classifiers.c.name == project.name))) request.db.query(Release).filter(Release.name == project.name).delete() request.db.query(Project).filter(Project.name == project.name).delete() request.session.flash( f"Successfully blacklisted {project_name!r}", queue="success", ) return HTTPSeeOther(request.route_path("admin.blacklist.list"))
def file_upload(request): # Before we do anything, if there isn't an authenticated user with this # request, then we'll go ahead and bomb out. if request.authenticated_userid is None: raise _exc_with_message( HTTPForbidden, "Invalid or non-existent authentication information.", ) # distutils "helpfully" substitutes unknown, but "required" values with the # string "UNKNOWN". This is basically never what anyone actually wants so # we'll just go ahead and delete anything whose value is UNKNOWN. for key in list(request.POST): if request.POST.get(key) == "UNKNOWN": del request.POST[key] # We require protocol_version 1, it's the only supported version however # passing a different version should raise an error. if request.POST.get("protocol_version", "1") != "1": raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.") # Look up all of the valid classifiers all_classifiers = request.db.query(Classifier).all() # Validate and process the incoming metadata. form = MetadataForm(request.POST) form.classifiers.choices = [ (c.classifier, c.classifier) for c in all_classifiers ] if not form.validate(): for field_name in _error_message_order: if field_name in form.errors: break else: field_name = sorted(form.errors.keys())[0] raise _exc_with_message( HTTPBadRequest, "{field}: {msgs[0]}".format( field=field_name, msgs=form.errors[field_name], ), ) # TODO: We need a better method of blocking names rather than just # hardcoding some names into source control. if form.name.data.lower() in {"requirements.txt", "rrequirements.txt"}: raise _exc_with_message( HTTPBadRequest, "The name {!r} is not allowed.".format(form.name.data), ) # Ensure that we have file data in the request. if "content" not in request.POST: raise _exc_with_message( HTTPBadRequest, "Upload payload does not have a file.", ) # Look up the project first before doing anything else, this is so we can # automatically register it if we need to and can check permissions before # going any further. try: project = ( request.db.query(Project) .filter( Project.normalized_name == func.normalize_pep426_name(form.name.data)).one() ) except NoResultFound: # The project doesn't exist in our database, so we'll add it along with # a role setting the current user as the "Owner" of the project. project = Project(name=form.name.data) request.db.add(project) request.db.add( Role(user=request.user, project=project, role_name="Owner") ) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=project.name, action="create", submitted_by=request.user, submitted_from=request.client_addr, ), ) request.db.add( JournalEntry( name=project.name, action="add Owner {}".format(request.user.username), submitted_by=request.user, submitted_from=request.client_addr, ), ) # Check that the user has permission to do things to this project, if this # is a new project this will act as a sanity check for the role we just # added above. if not request.has_permission("upload", project): raise _exc_with_message( HTTPForbidden, "You are not allowed to upload to {!r}.".format(project.name) ) try: release = ( request.db.query(Release) .filter( (Release.project == project) & (Release.version == form.version.data)).one() ) except NoResultFound: release = Release( project=project, _classifiers=[ c for c in all_classifiers if c.classifier in form.classifiers.data ], _pypi_hidden=False, dependencies=list(_construct_dependencies( form, { "requires": DependencyKind.requires, "provides": DependencyKind.provides, "obsoletes": DependencyKind.obsoletes, "requires_dist": DependencyKind.requires_dist, "provides_dist": DependencyKind.provides_dist, "obsoletes_dist": DependencyKind.obsoletes_dist, "requires_external": DependencyKind.requires_external, "project_urls": DependencyKind.project_url, } )), **{ k: getattr(form, k).data for k in { # This is a list of all the fields in the form that we # should pull off and insert into our new release. "version", "summary", "description", "license", "author", "author_email", "maintainer", "maintainer_email", "keywords", "platform", "home_page", "download_url", "requires_python", } } ) request.db.add(release) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="new release", submitted_by=request.user, submitted_from=request.client_addr, ), ) # TODO: We need a better solution to this than to just do it inline inside # this method. Ideally the version field would just be sortable, but # at least this should be some sort of hook or trigger. releases = ( request.db.query(Release) .filter(Release.project == project) .all() ) for i, r in enumerate(sorted( releases, key=lambda x: packaging.version.parse(x.version))): r._pypi_ordering = i # TODO: Again, we should figure out a better solution to doing this than # just inlining this inside this method. if project.autohide: for r in releases: r._pypi_hidden = bool(not r == release) # Pull the filename out of our POST data. filename = request.POST["content"].filename # Make sure that the filename does not contain any path separators. if "/" in filename or "\\" in filename: raise _exc_with_message( HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name.", ) # Make sure the filename ends with an allowed extension. if _dist_file_re.search(filename) is None: raise _exc_with_message(HTTPBadRequest, "Invalid file extension.") # Make sure that our filename matches the project that it is being uploaded # to. prefix = pkg_resources.safe_name(project.name).lower() if not pkg_resources.safe_name(filename).lower().startswith(prefix): raise _exc_with_message( HTTPBadRequest, "The filename for {!r} must start with {!r}.".format( project.name, prefix, ) ) # Check the content type of what is being uploaded if (not request.POST["content"].type or request.POST["content"].type.startswith("image/")): raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.") # Check to see if the file that was uploaded exists already or not. if request.db.query( request.db.query(File) .filter(File.filename == filename) .exists()).scalar(): raise _exc_with_message(HTTPBadRequest, "File already exists.") # Check to see if the file that was uploaded exists in our filename log. if (request.db.query( request.db.query(Filename) .filter(Filename.filename == filename) .exists()).scalar()): raise _exc_with_message( HTTPBadRequest, "This filename has previously been used, you should use a " "different version.", ) # The project may or may not have a file size specified on the project, if # it does then it may or may not be smaller or larger than our global file # size limits. file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit])) with tempfile.TemporaryDirectory() as tmpdir: temporary_filename = os.path.join(tmpdir, filename) # Buffer the entire file onto disk, checking the hash of the file as we # go along. with open(temporary_filename, "wb") as fp: file_size = 0 file_hashes = { "md5": hashlib.md5(), "sha256": hashlib.sha256(), "blake2_256": blake2b(digest_size=256 // 8), } for chunk in iter( lambda: request.POST["content"].file.read(8096), b""): file_size += len(chunk) if file_size > file_size_limit: raise _exc_with_message(HTTPBadRequest, "File too large.") fp.write(chunk) for hasher in file_hashes.values(): hasher.update(chunk) # Take our hash functions and compute the final hashes for them now. file_hashes = { k: h.hexdigest().lower() for k, h in file_hashes.items() } # Actually verify the digests that we've gotten. We're going to use # hmac.compare_digest even though we probably don't actually need to # because it's better safe than sorry. In the case of multiple digests # we expect them all to be given. if not all([ hmac.compare_digest( getattr(form, "{}_digest".format(digest_name)).data.lower(), digest_value, ) for digest_name, digest_value in file_hashes.items() if getattr(form, "{}_digest".format(digest_name)).data ]): raise _exc_with_message( HTTPBadRequest, "The digest supplied does not match a digest calculated " "from the uploaded file." ) # Check the file to make sure it is a valid distribution file. if not _is_valid_dist_file(temporary_filename, form.filetype.data): raise _exc_with_message( HTTPBadRequest, "Invalid distribution file.", ) # Check that if it's a binary wheel, it's on a supported platform if filename.endswith(".whl"): wheel_info = _wheel_file_re.match(filename) plats = wheel_info.group("plat").split(".") for plat in plats: if not _valid_platform_tag(plat): raise _exc_with_message( HTTPBadRequest, "Binary wheel '{filename}' has an unsupported " "platform tag '{plat}'." .format(filename=filename, plat=plat) ) # Also buffer the entire signature file to disk. if "gpg_signature" in request.POST: has_signature = True with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp: signature_size = 0 for chunk in iter( lambda: request.POST["gpg_signature"].file.read(8096), b""): signature_size += len(chunk) if signature_size > MAX_SIGSIZE: raise _exc_with_message( HTTPBadRequest, "Signature too large.", ) fp.write(chunk) # Check whether signature is ASCII armored with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp: if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"): raise _exc_with_message( HTTPBadRequest, "PGP signature is not ASCII armored.", ) else: has_signature = False # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add(Filename(filename=filename)) # Store the information about the file in the database. file_ = File( release=release, filename=filename, python_version=form.pyversion.data, packagetype=form.filetype.data, comment_text=form.comment.data, size=file_size, has_signature=bool(has_signature), md5_digest=file_hashes["md5"], sha256_digest=file_hashes["sha256"], blake2_256_digest=file_hashes["blake2_256"], # Figure out what our filepath is going to be, we're going to use a # directory structure based on the hash of the file contents. This # will ensure that the contents of the file cannot change without # it also changing the path that the file is saved too. path="/".join([ file_hashes[PATH_HASHER][:2], file_hashes[PATH_HASHER][2:4], file_hashes[PATH_HASHER][4:], filename, ]), ) request.db.add(file_) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="add {python_version} file {filename}".format( python_version=file_.python_version, filename=file_.filename, ), submitted_by=request.user, submitted_from=request.client_addr, ), ) # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is # committed. storage = request.find_service(IFileStorage) storage.store( file_.path, os.path.join(tmpdir, filename), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) if has_signature: storage.store( file_.pgp_path, os.path.join(tmpdir, filename + ".asc"), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) # TODO: Once we no longer have the legacy code base running PyPI we can # go ahead and delete this tiny bit of shim code, since it only # exists to purge stuff on legacy PyPI when uploaded to Warehouse old_domain = request.registry.settings.get("warehouse.legacy_domain") if old_domain: request.tm.get().addAfterCommitHook( _legacy_purge, args=["https://{}/pypi".format(old_domain)], kws={"data": {":action": "purge", "project": project.name}}, ) return Response()
def verify_project_role(request): token_service = request.find_service(ITokenService, name="email") user_service = request.find_service(IUserService, context=None) def _error(message): request.session.flash(message, queue="error") return HTTPSeeOther(request.route_path("manage.projects")) try: token = request.params.get("token") data = token_service.loads(token) except TokenExpired: return _error(request._("Expired token: request a new project role invite")) except TokenInvalid: return _error(request._("Invalid token: request a new project role invite")) except TokenMissing: return _error(request._("Invalid token: no token supplied")) # Check whether this token is being used correctly if data.get("action") != "email-project-role-verify": return _error(request._("Invalid token: not a collaboration invitation token")) user = user_service.get_user(data.get("user_id")) if user != request.user: return _error(request._("Role invitation is not valid.")) project = ( request.db.query(Project).filter(Project.id == data.get("project_id")).one() ) desired_role = data.get("desired_role") role_invite = ( request.db.query(RoleInvitation) .filter(RoleInvitation.project == project) .filter(RoleInvitation.user == user) .one_or_none() ) if not role_invite: return _error(request._("Role invitation no longer exists.")) # Use the renderer to bring up a confirmation page # before adding as contributor if request.method == "GET": return { "project_name": project.name, "desired_role": desired_role, } elif request.method == "POST" and "decline" in request.POST: request.db.delete(role_invite) request.session.flash( request._( "Invitation for '${project_name}' is declined.", mapping={"project_name": project.name}, ), queue="success", ) return HTTPSeeOther(request.route_path("manage.projects")) request.db.add(Role(user=user, project=project, role_name=desired_role)) request.db.delete(role_invite) request.db.add( JournalEntry( name=project.name, action=f"accepted {desired_role} {user.username}", submitted_by=request.user, submitted_from=request.remote_addr, ) ) project.record_event( tag="project:role:accepted", ip_address=request.remote_addr, additional={ "submitted_by": request.user.username, "role_name": desired_role, "target_user": user.username, }, ) user.record_event( tag="account:role:accepted", ip_address=request.remote_addr, additional={ "submitted_by": request.user.username, "project_name": project.name, "role_name": desired_role, }, ) owner_roles = ( request.db.query(Role) .filter(Role.project == project) .filter(Role.role_name == "Owner") .all() ) owner_users = {owner.user for owner in owner_roles} # Don't send email to new user if they are now an owner owner_users.discard(user) submitter_user = user_service.get_user(data.get("submitter_id")) send_collaborator_added_email( request, owner_users, user=user, submitter=submitter_user, project_name=project.name, role=desired_role, ) send_added_as_collaborator_email( request, user, submitter=submitter_user, project_name=project.name, role=desired_role, ) request.session.flash( request._( "You are now ${role} of the '${project_name}' project.", mapping={"project_name": project.name, "role": desired_role}, ), queue="success", ) if desired_role == "Owner": return HTTPSeeOther( request.route_path("manage.project.roles", project_name=project.name) ) else: return HTTPSeeOther(request.route_path("packaging.project", name=project.name))