def release_prohibited_project_name(request): project_name = request.POST.get("project_name") if project_name is None: request.session.flash("Provide a project name", queue="error") return HTTPSeeOther(request.current_route_path()) prohibited_project_name = ( request.db.query(ProhibitedProjectName) .filter(ProhibitedProjectName.name == func.normalize_pep426_name(project_name)) .first() ) if prohibited_project_name is None: request.session.flash( f"{project_name!r} does not exist on prohibited project name list.", queue="error", ) return HTTPSeeOther(request.current_route_path()) project = ( request.db.query(Project) .filter(Project.normalized_name == func.normalize_pep426_name(project_name)) .first() ) if project is not None: request.session.flash( f"{project_name!r} exists and is not on the prohibited project name list.", queue="error", ) return HTTPSeeOther(request.current_route_path()) username = request.POST.get("username") if not username: request.session.flash("Provide a username", queue="error") return HTTPSeeOther(request.current_route_path()) user = request.db.query(User).filter(User.username == username).first() if user is None: request.session.flash(f"Unknown username '{username}'", queue="error") return HTTPSeeOther(request.current_route_path()) project = Project(name=project_name) request.db.add(project) request.db.add(Role(project=project, user=user, role_name="Owner")) request.db.delete(prohibited_project_name) request.session.flash( f"{project.name!r} released to {user.username!r}.", queue="success" ) request.db.flush() return HTTPSeeOther( request.route_path("admin.project.detail", project_name=project.normalized_name) )
def release_urls(request, package_name: str, version: str): files = ( request.db.query(File) .join(Release, Project) .filter( (Project.normalized_name == func.normalize_pep426_name(package_name)) & (Release.version == version) ) .all() ) return [ { "filename": f.filename, "packagetype": f.packagetype, "python_version": f.python_version, "size": f.size, "md5_digest": f.md5_digest, "sha256_digest": f.sha256_digest, "digests": {"md5": f.md5_digest, "sha256": f.sha256_digest}, "has_sig": f.has_signature, "upload_time": f.upload_time.isoformat() + "Z", "comment_text": f.comment_text, # TODO: Remove this once we've had a long enough time with it # here to consider it no longer in use. "downloads": -1, "path": f.path, "url": request.route_url("packaging.file", path=f.path), } for f in files ]
def __getitem__(self, project): try: return self.request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name(project) ).one() except NoResultFound: raise KeyError from None
def release_urls(request, package_name, version): files = (request.db.query(File).join(Release, Project).filter( (Project.normalized_name == func.normalize_pep426_name(package_name)) & (Release.version == version)).all()) return [ { "filename": f.filename, "packagetype": f.packagetype, "python_version": f.python_version, "size": f.size, "md5_digest": f.md5_digest, "sha256_digest": f.sha256_digest, "digests": { "md5": f.md5_digest, "sha256": f.sha256_digest, }, "has_sig": f.has_signature, "upload_time": f.upload_time.isoformat() + "Z", "comment_text": f.comment_text, # TODO: Remove this once we've had a long enough time with it # here to consider it no longer in use. "downloads": -1, "path": f.path, "url": request.route_url("packaging.file", path=f.path), } for f in files ]
def package_roles(request, package_name): roles = (request.db.query(Role).join( User, Project).filter(Project.normalized_name == func.normalize_pep426_name( package_name)).order_by(Role.role_name.desc(), User.username).all()) return [(r.role_name, r.user.username) for r in roles]
def blacklist(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None blacklist_query = request.db.query(BlacklistedProject).order_by( BlacklistedProject.name ) if q: terms = shlex.split(q) filters = [] for term in terms: filters.append( BlacklistedProject.name.ilike(func.normalize_pep426_name(term)) ) blacklist_query = blacklist_query.filter(or_(*filters)) blacklist = SQLAlchemyORMPage( blacklist_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"blacklist": blacklist, "query": q}
def add_blacklist(request): project_name = request.POST.get("project") if project_name is None: raise HTTPBadRequest("Have a project to confirm.") comment = request.POST.get("comment", "") # Verify that the user has confirmed the request to blacklist. confirm = request.POST.get("confirm") if not confirm: request.session.flash("Confirm the blacklist request", queue="error") return HTTPSeeOther(request.current_route_path()) elif canonicalize_name(confirm) != canonicalize_name(project_name): request.session.flash( f"{confirm!r} is not the same as {project_name!r}", queue="error") return HTTPSeeOther(request.current_route_path()) # Add our requested blacklist. request.db.add( BlacklistedProject(name=project_name, comment=comment, blacklisted_by=request.user)) # Go through and delete the project and everything related to it so that # our blacklist actually blocks things and isn't ignored (since the # blacklist only takes effect on new project registration). project = (request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name( project_name)).first()) if project is not None: remove_project(project, request) request.session.flash(f"Blacklisted {project_name!r}", queue="success") return HTTPSeeOther(request.route_path("admin.blacklist.list"))
def release_urls(request, package_name, version): files = ( request.db.query(File) .join(Release, Project) .filter((Project.normalized_name == func.normalize_pep426_name(package_name)) & (Release.version == version)) .all() ) return [ { "filename": f.filename, "packagetype": f.packagetype, "python_version": f.python_version, "size": f.size, "md5_digest": f.md5_digest, "has_sig": f.has_signature, "upload_time": f.upload_time, "comment_text": f.comment_text, "downloads": f.downloads, "url": request.route_url("packaging.file", path=f.path), } for f in files ]
def release_data(request, package_name, version): try: release = ( request.db.query(Release) .options(orm.undefer("description")) .join(Project) .filter((Project.normalized_name == func.normalize_pep426_name(package_name)) & (Release.version == version)) .one() ) except NoResultFound: return {} stats_svc = request.find_service(IDownloadStatService) return { "name": release.project.name, "version": release.version, "stable_version": release.project.stable_version, "bugtrack_url": release.project.bugtrack_url, "package_url": request.route_url( "packaging.project", name=release.project.name, ), "release_url": request.route_url( "packaging.release", name=release.project.name, version=release.version, ), "docs_url": release.project.documentation_url, "home_page": release.home_page, "download_url": release.download_url, "project_url": list(release.project_urls), "author": release.author, "author_email": release.author_email, "maintainer": release.maintainer, "maintainer_email": release.maintainer_email, "summary": release.summary, "description": release.description, "license": release.license, "keywords": release.keywords, "platform": release.platform, "classifiers": list(release.classifiers), "requires": list(release.requires), "requires_dist": list(release.requires_dist), "provides": list(release.provides), "provides_dist": list(release.provides_dist), "obsoletes": list(release.obsoletes), "obsoletes_dist": list(release.obsoletes_dist), "requires_python": release.requires_python, "requires_external": list(release.requires_external), "_pypi_ordering": release._pypi_ordering, "_pypi_hidden": release._pypi_hidden, "downloads": { "last_day": stats_svc.get_daily_stats(release.project.name), "last_week": stats_svc.get_weekly_stats(release.project.name), "last_month": stats_svc.get_monthly_stats(release.project.name), }, }
def __getitem__(self, project): try: return (self.request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name( project)).one()) except NoResultFound: raise KeyError from None
def _get_project(request, vuln_report: vulnerabilities.VulnerabilityReportRequest): return ( request.db.query(Project) .filter( Project.normalized_name == func.normalize_pep426_name(vuln_report.project) ) .one() )
def package_roles(request, package_name: str): roles = ( request.db.query(Role) .join(User, Project) .filter(Project.normalized_name == func.normalize_pep426_name(package_name)) .order_by(Role.role_name.desc(), User.username) .all() ) return [(r.role_name, r.user.username) for r in roles]
def package_releases(request, package_name, show_hidden=False): # This used to support the show_hidden parameter to determine if it should # show hidden releases or not. However, Warehouse doesn't support the # concept of hidden releases, so it is just no-opd now and left here for # compatibility's sake. versions = (request.db.query(Release.version).join(Project).filter( Project.normalized_name == func.normalize_pep426_name( package_name)).order_by(Release._pypi_ordering).all()) return [v[0] for v in versions]
def package_hosting_mode(request, package_name): try: project = (request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name( package_name)).one()) except NoResultFound: return None else: return project.hosting_mode
def release_data(request, package_name, version): try: release = ( request.db.query(Release) .options(orm.undefer("description")) .join(Project) .filter( (Project.normalized_name == func.normalize_pep426_name(package_name)) & (Release.version == version) ) .one() ) except NoResultFound: return {} return { "name": release.project.name, "version": release.version, "stable_version": release.project.stable_version, "bugtrack_url": release.project.bugtrack_url, "package_url": request.route_url( "packaging.project", name=release.project.name ), "release_url": request.route_url( "packaging.release", name=release.project.name, version=release.version ), "docs_url": release.project.documentation_url, "home_page": release.home_page, "download_url": release.download_url, "project_url": list(release.project_urls), "author": release.author, "author_email": release.author_email, "maintainer": release.maintainer, "maintainer_email": release.maintainer_email, "summary": release.summary, "description": release.description, "license": release.license, "keywords": release.keywords, "platform": release.platform, "classifiers": list(release.classifiers), "requires": list(release.requires), "requires_dist": list(release.requires_dist), "provides": list(release.provides), "provides_dist": list(release.provides_dist), "obsoletes": list(release.obsoletes), "obsoletes_dist": list(release.obsoletes_dist), "requires_python": release.requires_python, "requires_external": list(release.requires_external), "_pypi_ordering": release._pypi_ordering, "_pypi_hidden": release._pypi_hidden, "downloads": {"last_day": -1, "last_week": -1, "last_month": -1}, "cheesecake_code_kwalitee_id": None, "cheesecake_documentation_id": None, "cheesecake_installability_id": None, }
def add_prohibited_project_names(request): project_name = request.POST.get("project") if project_name is None: raise HTTPBadRequest("Have a project to confirm.") comment = request.POST.get("comment", "") # Verify that the user has confirmed the request to prohibit. confirm = request.POST.get("confirm") if not confirm: request.session.flash( "Confirm the prohibited project name request", queue="error" ) return HTTPSeeOther(request.current_route_path()) elif canonicalize_name(confirm) != canonicalize_name(project_name): request.session.flash( f"{confirm!r} is not the same as {project_name!r}", queue="error" ) return HTTPSeeOther(request.current_route_path()) # Check to make sure the object doesn't already exist. if ( request.db.query(literal(True)) .filter( request.db.query(ProhibitedProjectName) .filter(ProhibitedProjectName.name == project_name) .exists() ) .scalar() ): request.session.flash( f"{project_name!r} has already been prohibited.", queue="error" ) return HTTPSeeOther(request.route_path("admin.prohibited_project_names.list")) # Add our requested prohibition. request.db.add( ProhibitedProjectName( name=project_name, comment=comment, prohibited_by=request.user ) ) # Go through and delete the project and everything related to it so that # our prohibition actually blocks things and isn't ignored (since the # prohibition only takes effect on new project registration). project = ( request.db.query(Project) .filter(Project.normalized_name == func.normalize_pep426_name(project_name)) .first() ) if project is not None: remove_project(project, request) request.session.flash(f"Prohibited Project Name {project_name!r}", queue="success") return HTTPSeeOther(request.route_path("admin.prohibited_project_names.list"))
def release_data(request, package_name: str, version: str): try: release = ( request.db.query(Release) .options(orm.undefer("description")) .join(Project) .filter( (Project.normalized_name == func.normalize_pep426_name(package_name)) & (Release.version == version) ) .one() ) except NoResultFound: return {} return { "name": release.project.name, "version": release.version, "stable_version": None, "bugtrack_url": None, "package_url": request.route_url( "packaging.project", name=release.project.name ), "release_url": request.route_url( "packaging.release", name=release.project.name, version=release.version ), "docs_url": _clean_for_xml(release.project.documentation_url), "home_page": _clean_for_xml(release.home_page), "download_url": _clean_for_xml(release.download_url), "project_url": [_clean_for_xml(url) for url in release.project_urls], "author": _clean_for_xml(release.author), "author_email": _clean_for_xml(release.author_email), "maintainer": _clean_for_xml(release.maintainer), "maintainer_email": _clean_for_xml(release.maintainer_email), "summary": _clean_for_xml(release.summary), "description": _clean_for_xml(release.description), "license": _clean_for_xml(release.license), "keywords": _clean_for_xml(release.keywords), "platform": release.platform, "classifiers": list(release.classifiers), "requires": list(release.requires), "requires_dist": list(release.requires_dist), "provides": list(release.provides), "provides_dist": list(release.provides_dist), "obsoletes": list(release.obsoletes), "obsoletes_dist": list(release.obsoletes_dist), "requires_python": release.requires_python, "requires_external": list(release.requires_external), "_pypi_ordering": release._pypi_ordering, "downloads": {"last_day": -1, "last_week": -1, "last_month": -1}, "cheesecake_code_kwalitee_id": None, "cheesecake_documentation_id": None, "cheesecake_installability_id": None, }
def __getitem__(self, organization): # Try returning organization with matching name. try: return (self.request.db.query(Organization).filter( Organization.normalized_name == func.normalize_pep426_name( organization)).one()) except NoResultFound: pass # Try redirecting to a renamed organization. try: organization = (self.request.db.query(Organization).join( OrganizationNameCatalog, OrganizationNameCatalog.organization_id == Organization.id, ).filter(OrganizationNameCatalog.normalized_name == func.normalize_pep426_name(organization)).one()) raise HTTPPermanentRedirect( self.request.matched_route.generate( {"organization_name": organization.normalized_name})) except NoResultFound: raise KeyError from None
def package_hosting_mode(request, package_name): try: project = ( request.db.query(Project) .filter(Project.normalized_name == func.normalize_pep426_name(package_name)) .one() ) except NoResultFound: return None else: return project.hosting_mode
def package_releases(request, package_name, show_hidden=False): # This used to support the show_hidden parameter to determine if it should # show hidden releases or not. However, Warehouse doesn't support the # concept of hidden releases, so this parameter controls if the latest # version or all_versions are returned. project = (request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name( package_name)).one()) if show_hidden: return [v.version for v in project.all_versions] else: return [project.latest_version.version]
def add_blacklist(request): project_name = request.POST.get("project") if project_name is None: raise HTTPBadRequest("Have a project to confirm.") comment = request.POST.get("comment", "") # Verify that the user has confirmed the request to blacklist. confirm = request.POST.get("confirm") if not confirm: request.session.flash("Confirm the blacklist request", queue="error") return HTTPSeeOther(request.current_route_path()) elif canonicalize_name(confirm) != canonicalize_name(project_name): request.session.flash( f"{confirm!r} is not the same as {project_name!r}", queue="error" ) return HTTPSeeOther(request.current_route_path()) # Check to make sure the object doesn't already exist. if ( request.db.query(literal(True)) .filter( request.db.query(BlacklistedProject) .filter(BlacklistedProject.name == project_name) .exists() ) .scalar() ): request.session.flash( f"{project_name!r} has already been blacklisted.", queue="error" ) return HTTPSeeOther(request.route_path("admin.blacklist.list")) # Add our requested blacklist. request.db.add( BlacklistedProject( name=project_name, comment=comment, blacklisted_by=request.user ) ) # Go through and delete the project and everything related to it so that # our blacklist actually blocks things and isn't ignored (since the # blacklist only takes effect on new project registration). project = ( request.db.query(Project) .filter(Project.normalized_name == func.normalize_pep426_name(project_name)) .first() ) if project is not None: remove_project(project, request) request.session.flash(f"Blacklisted {project_name!r}", queue="success") return HTTPSeeOther(request.route_path("admin.blacklist.list"))
def confirm_blacklist(request): project_name = request.GET.get("project") if project_name is None: raise HTTPBadRequest("Have a project to confirm.") comment = request.GET.get("comment", "") # We need to look up to see if there is an existing project, releases, # files, roles, etc for what we're attempting to blacklist. If there is we # need to warn that blacklisting will delete those. project = ( request.db.query(Project) .filter(Project.normalized_name == func.normalize_pep426_name(project_name)) .first() ) if project is not None: releases = ( request.db.query(Release) .join(Project) .filter(Release.project == project) .all() ) files = ( request.db.query(File) .join(Release) .join(Project) .filter(Release.project == project) .all() ) roles = ( request.db.query(Role) .join(User) .join(Project) .filter(Role.project == project) .distinct(User.username) .order_by(User.username) .all() ) else: releases = [] files = [] roles = [] return { "blacklist": {"project": project_name, "comment": comment}, "existing": { "project": project, "releases": releases, "files": files, "roles": roles, }, }
def package_releases(request, package_name, show_hidden=False): # This used to support the show_hidden parameter to determine if it should # show hidden releases or not. However, Warehouse doesn't support the # concept of hidden releases, so it is just no-opd now and left here for # compatibility's sake. versions = ( request.db.query(Release.version) .join(Project) .filter(Project.normalized_name == func.normalize_pep426_name(package_name)) .order_by(Release._pypi_ordering) .all() ) return [v[0] for v in versions]
class Project(db.ModelBase): __tablename__ = "packages" __table_args__ = (CheckConstraint( "name ~* '^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$'::text", name="packages_valid_name", ), ) __repr__ = make_repr("name") name = Column(Text, primary_key=True, nullable=False) normalized_name = orm.column_property(func.normalize_pep426_name(name)) stable_version = Column(Text) autohide = Column(Boolean, server_default=sql.true()) comments = Column(Boolean, server_default=sql.true()) bugtrack_url = Column(Text) hosting_mode = Column(Text, nullable=False, server_default="pypi-explicit") created = Column( DateTime(timezone=False), nullable=False, server_default=sql.func.now(), ) releases = orm.relationship( "Release", backref="project", cascade="all, delete-orphan", lazy="dynamic", ) def __getitem__(self, version): try: return self.releases.filter(Release.version == version).one() except NoResultFound: raise KeyError from None @property def documentation_url(self): # TODO: Move this into the database and elimnate the use of the # threadlocal here. registry = get_current_registry() request = get_current_request() path = "/".join([self.name, "index.html"]) # If the path doesn't exist, then we'll just return a None here. if not registry["filesystems"]["documentation"].exists(path): return return request.route_url("legacy.docs", project=self.name)
def find_organizationid(self, name): """ Find the unique organization identifier for the given normalized name or None if there is no organization with the given name. """ normalized_name = func.normalize_pep426_name(name) try: (organization_id, ) = (self.db.query( OrganizationNameCatalog.organization_id).filter( OrganizationNameCatalog.normalized_name == normalized_name).one()) except NoResultFound: return return organization_id
def bulk_add_prohibited_project_names(request): if request.method == "POST": project_names = request.POST.get("projects", "").split() comment = request.POST.get("comment", "") for project_name in project_names: # Check to make sure the object doesn't already exist. if ( request.db.query(literal(True)) .filter( request.db.query(ProhibitedProjectName) .filter(ProhibitedProjectName.name == project_name) .exists() ) .scalar() ): continue # Add our requested prohibition. request.db.add( ProhibitedProjectName( name=project_name, comment=comment, prohibited_by=request.user ) ) # Go through and delete the project and everything related to it so that # our prohibition actually blocks things and isn't ignored (since the # prohibition only takes effect on new project registration). project = ( request.db.query(Project) .filter( Project.normalized_name == func.normalize_pep426_name(project_name) ) .first() ) if project is not None: remove_project(project, request, flash=False) request.session.flash( f"Prohibited {len(project_names)!r} projects", queue="success" ) return HTTPSeeOther( request.route_path("admin.prohibited_project_names.bulk_add") ) return {}
def release_urls(request, package_name, version): files = (request.db.query(File).join(Release, Project).filter( (Project.normalized_name == func.normalize_pep426_name(package_name)) & (Release.version == version)).all()) return [{ "filename": f.filename, "packagetype": f.packagetype, "python_version": f.python_version, "size": f.size, "md5_digest": f.md5_digest, "has_sig": f.has_signature, "upload_time": f.upload_time, "comment_text": f.comment_text, "downloads": f.downloads, "url": request.route_url("packaging.file", path=f.path), } for f in files]
def package_releases(request, package_name: str, show_hidden: bool = False): try: project = ( request.db.query(Project) .filter(Project.normalized_name == func.normalize_pep426_name(package_name)) .one() ) except NoResultFound: return [] # This used to support the show_hidden parameter to determine if it should # show hidden releases or not. However, Warehouse doesn't support the # concept of hidden releases, so this parameter controls if the latest # version or all_versions are returned. if show_hidden: return [v.version for v in project.all_versions] else: latest_version = project.latest_version if latest_version is None: return [] return [latest_version.version]
class Project(db.Model, Syncable): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.Text, nullable=False, index=True) team_slug = db.Column(db.String(255)) normalized_name = orm.column_property(func.normalize_pep426_name(name)) description = db.Column(db.Text) html_url = db.Column(db.String(255)) subscribers_count = db.Column(db.SmallInteger, default=0, nullable=False) stargazers_count = db.Column(db.SmallInteger, default=0, nullable=False) forks_count = db.Column(db.SmallInteger, default=0, nullable=False) open_issues_count = db.Column(db.SmallInteger, default=0, nullable=False) is_active = db.Column(db.Boolean, default=True, nullable=False, index=True) transfer_issue_url = db.Column(db.String(255)) membership = db.relationship("ProjectMembership", backref="project", lazy="dynamic") credentials = db.relationship("ProjectCredential", backref="project", lazy="dynamic") uploads = db.relationship( "ProjectUpload", backref="project", lazy="dynamic", order_by=lambda: ProjectUpload.ordering.desc().nullslast(), ) created_at = db.Column(db.DateTime, nullable=True) updated_at = db.Column(db.DateTime, nullable=True) pushed_at = db.Column(db.DateTime, nullable=True) __tablename__ = "projects" __table_args__ = ( db.Index("release_name_idx", "name"), db.Index("release_name_is_active_idx", "name", "is_active"), ) def __str__(self): return self.name @aggregated("uploads", db.Column(db.SmallInteger)) def uploads_count(self): return db.func.count("1") @aggregated("membership", db.Column(db.SmallInteger)) def membership_count(self): return db.func.count("1") @property def current_user_is_member(self): if not current_user: return False elif not current_user.is_authenticated: return False elif current_user_is_roadie(): return True else: return self.user_is_member(current_user) @property def current_user_is_lead(self): if not current_user: return False elif not current_user.is_authenticated: return False elif current_user_is_roadie(): return True else: return current_user.id in [ user.id for user in self.lead_members.options(orm.load_only("id")) ] @property def all_members(self): return (User.active_members().join(User.projects_memberships).filter( ProjectMembership.project_id == self.id)) @property def nonlead_members(self): return self.all_members.filter(ProjectMembership.is_lead.is_(False)) @property def lead_members(self): return self.all_members.filter(ProjectMembership.is_lead.is_(True)) def user_is_member(self, user): return user.id in [ member.id for member in self.all_members.options(orm.load_only("id")) ] @property def pypi_json_url(self): """ The URL to fetch JSON data from PyPI, using a timestamp to work-around the PyPI CDN cache. """ return ( f"https://pypi.org/pypi/{self.normalized_name}/json?time={int(time.time())}" ) def create_transfer_issue(self, assignees, **data): issue_response = github.new_project_issue( repo=self.name, data={ "title": render_template("hooks/project-title.txt", **data), "body": render_template("hooks/project-body.txt", **data), "assignees": assignees, }, ) issue_data = issue_response.json() issue_url = issue_data.get("html_url") if issue_url.startswith(f"https://github.com/jazzband/{self.name}"): self.transfer_issue_url = issue_url self.save() def create_team(self): team_response = github.create_project_team(self.name) if team_response and team_response.status_code == 201: team_data = team_response.json() self.team_slug = team_data.get("slug") self.save() return team_response
class Project(db.Model, Helpers, Syncable): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.Text, nullable=False, index=True) normalized_name = orm.column_property(func.normalize_pep426_name(name)) description = db.Column(db.Text) html_url = db.Column(db.String(255)) subscribers_count = db.Column(db.SmallInteger, default=0, nullable=False) stargazers_count = db.Column(db.SmallInteger, default=0, nullable=False) forks_count = db.Column(db.SmallInteger, default=0, nullable=False) open_issues_count = db.Column(db.SmallInteger, default=0, nullable=False) uploads_count = db.Column(db.SmallInteger, default=0) is_active = db.Column(db.Boolean, default=True, nullable=False, index=True) membership = db.relationship('ProjectMembership', backref='project', lazy='dynamic') credentials = db.relationship('ProjectCredential', backref='project', lazy='dynamic') uploads = db.relationship('ProjectUpload', backref='project', lazy='dynamic') created_at = db.Column(db.DateTime, nullable=True) updated_at = db.Column(db.DateTime, nullable=True) pushed_at = db.Column(db.DateTime, nullable=True) __tablename__ = 'projects' __table_args__ = ( db.Index('release_name_idx', 'name'), db.Index('release_name_is_active_idx', 'name', 'is_active'), ) def __str__(self): return self.name def __repr__(self): return '<Project %s: %s (%s)>' % (self.id, self.name, self.id) @aggregated('uploads', db.Column(db.SmallInteger)) def uploads_count(self): return db.func.count('1') @property def current_user_is_member(self): if not current_user: return False elif not current_user.is_authenticated: return False elif current_user_is_roadie(): return True else: return current_user.id in self.member_ids @property def member_ids(self): return [member.user.id for member in self.membership.all()] @property def pypi_json_url(self): return f'https://pypi.org/pypi/{self.normalized_name}/json' # noqa
def file_upload(request): # Before we do anything, if there isn't an authenticated user with this # request, then we'll go ahead and bomb out. if request.authenticated_userid is None: raise _exc_with_message( HTTPForbidden, "Invalid or non-existent authentication information.", ) # distutils "helpfully" substitutes unknown, but "required" values with the # string "UNKNOWN". This is basically never what anyone actually wants so # we'll just go ahead and delete anything whose value is UNKNOWN. for key in list(request.POST): if request.POST.get(key) == "UNKNOWN": del request.POST[key] # We require protocol_version 1, it's the only supported version however # passing a different version should raise an error. if request.POST.get("protocol_version", "1") != "1": raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.") # Look up all of the valid classifiers all_classifiers = request.db.query(Classifier).all() # Validate and process the incoming metadata. form = MetadataForm(request.POST) form.classifiers.choices = [ (c.classifier, c.classifier) for c in all_classifiers ] if not form.validate(): for field_name in _error_message_order: if field_name in form.errors: break else: field_name = sorted(form.errors.keys())[0] raise _exc_with_message( HTTPBadRequest, "{field}: {msgs[0]}".format( field=field_name, msgs=form.errors[field_name], ), ) # TODO: We need a better method of blocking names rather than just # hardcoding some names into source control. if form.name.data.lower() in {"requirements.txt", "rrequirements.txt"}: raise _exc_with_message( HTTPBadRequest, "The name {!r} is not allowed.".format(form.name.data), ) # Ensure that we have file data in the request. if "content" not in request.POST: raise _exc_with_message( HTTPBadRequest, "Upload payload does not have a file.", ) # Look up the project first before doing anything else, this is so we can # automatically register it if we need to and can check permissions before # going any further. try: project = ( request.db.query(Project) .filter( Project.normalized_name == func.normalize_pep426_name(form.name.data)).one() ) except NoResultFound: # The project doesn't exist in our database, so we'll add it along with # a role setting the current user as the "Owner" of the project. project = Project(name=form.name.data) request.db.add(project) request.db.add( Role(user=request.user, project=project, role_name="Owner") ) # Check that the user has permission to do things to this project, if this # is a new project this will act as a sanity check for the role we just # added above. if not request.has_permission("upload", project): raise _exc_with_message( HTTPForbidden, "You are not allowed to upload to {!r}.".format(project.name) ) try: release = ( request.db.query(Release) .filter( (Release.project == project) & (Release.version == form.version.data)).one() ) except NoResultFound: release = Release( project=project, _classifiers=[ c for c in all_classifiers if c.classifier in form.classifiers.data ], dependencies=list(_construct_dependencies( form, { "requires": DependencyKind.requires, "provides": DependencyKind.provides, "obsoletes": DependencyKind.obsoletes, "requires_dist": DependencyKind.requires_dist, "provides_dist": DependencyKind.provides_dist, "obsoletes_dist": DependencyKind.obsoletes_dist, "requires_external": DependencyKind.requires_external, "project_urls": DependencyKind.project_url, } )), **{ k: getattr(form, k).data for k in { # This is a list of all the fields in the form that we # should pull off and insert into our new release. "version", "summary", "description", "license", "author", "author_email", "maintainer", "maintainer_email", "keywords", "platform", "home_page", "download_url", "requires_python", } } ) request.db.add(release) # TODO: We need a better solution to this than to just do it inline inside # this method. Ideally the version field would just be sortable, but # at least this should be some sort of hook or trigger. releases = ( request.db.query(Release) .filter(Release.project == project) .all() ) for i, r in enumerate(sorted( releases, key=lambda x: packaging.version.parse(x.version))): r._pypi_ordering = i # Pull the filename out of our POST data. filename = request.POST["content"].filename # Make sure that the filename does not contain any path separators. if "/" in filename or "\\" in filename: raise _exc_with_message( HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name.", ) # Make sure the filename ends with an allowed extension. if _dist_file_re.search(filename) is None: raise _exc_with_message(HTTPBadRequest, "Invalid file extension.") # Make sure that our filename matches the project that it is being uploaded # to. prefix = pkg_resources.safe_name(project.name).lower() if not pkg_resources.safe_name(filename).lower().startswith(prefix): raise _exc_with_message( HTTPBadRequest, "The filename for {!r} must start with {!r}.".format( project.name, prefix, ) ) # Check to see if the file that was uploaded exists already or not. if request.db.query( request.db.query(File) .filter(File.filename == filename) .exists()).scalar(): raise _exc_with_message(HTTPBadRequest, "File already exists.") # Check to see if the file that was uploaded exists in our filename log. if (request.db.query( request.db.query(Filename) .filter(Filename.filename == filename) .exists()).scalar()): raise _exc_with_message( HTTPBadRequest, "This filename has previously been used, you should use a " "different version.", ) # The project may or may not have a file size specified on the project, if # it does then it may or may not be smaller or larger than our global file # size limits. file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit])) with tempfile.TemporaryDirectory() as tmpdir: # Buffer the entire file onto disk, checking the hash of the file as we # go along. with open(os.path.join(tmpdir, filename), "wb") as fp: file_size = 0 file_hash = hashlib.md5() for chunk in iter( lambda: request.POST["content"].file.read(8096), b""): file_size += len(chunk) if file_size > file_size_limit: raise _exc_with_message(HTTPBadRequest, "File too large.") fp.write(chunk) file_hash.update(chunk) # Actually verify that the md5 hash of the file matches the expected # md5 hash. We probably don't actually need to use hmac.compare_digest # here since both the md5_digest and the file whose file_hash we've # computed comes from the remote user, however better safe than sorry. if not hmac.compare_digest( form.md5_digest.data, file_hash.hexdigest()): raise _exc_with_message( HTTPBadRequest, "The MD5 digest supplied does not match a digest calculated " "from the uploaded file." ) # TODO: Check the file to make sure it is a valid distribution file. # Check that if it's a binary wheel, it's on a supported platform if filename.endswith(".whl"): wheel_info = _wheel_file_re.match(filename) plats = wheel_info.group("plat").split(".") if set(plats) - ALLOWED_PLATFORMS: raise _exc_with_message( HTTPBadRequest, "Binary wheel for an unsupported platform.", ) # Also buffer the entire signature file to disk. if "gpg_signature" in request.POST: has_signature = True with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp: signature_size = 0 for chunk in iter( lambda: request.POST["gpg_signature"].file.read(8096), b""): signature_size += len(chunk) if signature_size > MAX_SIGSIZE: raise _exc_with_message( HTTPBadRequest, "Signature too large.", ) fp.write(chunk) # Check whether signature is ASCII armored with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp: if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"): raise _exc_with_message( HTTPBadRequest, "PGP signature is not ASCII armored.", ) else: has_signature = False # TODO: We need some sort of trigger that will automatically add # filenames to Filename instead of relying on this code running # inside of our upload API. request.db.add(Filename(filename=filename)) # Store the information about the file in the database. file_ = File( release=release, filename=filename, python_version=form.pyversion.data, packagetype=form.filetype.data, comment_text=form.comment.data, size=file_size, has_signature=bool(has_signature), md5_digest=form.md5_digest.data, ) request.db.add(file_) # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is # committed. storage = request.find_service(IFileStorage) storage.store(file_.path, os.path.join(tmpdir, filename)) if has_signature: storage.store( file_.pgp_path, os.path.join(tmpdir, filename + ".asc"), ) return Response()
class Project(SitemapMixin, db.ModelBase): __tablename__ = "packages" __table_args__ = (CheckConstraint( "name ~* '^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$'::text", name="packages_valid_name", ), ) __repr__ = make_repr("name") name = Column(Text, primary_key=True, nullable=False) normalized_name = orm.column_property(func.normalize_pep426_name(name)) stable_version = Column(Text) autohide = Column(Boolean, server_default=sql.true()) comments = Column(Boolean, server_default=sql.true()) bugtrack_url = Column(Text) hosting_mode = Column(Text, nullable=False, server_default="pypi-only") created = Column( DateTime(timezone=False), nullable=False, server_default=sql.func.now(), ) has_docs = Column(Boolean) upload_limit = Column(Integer, nullable=True) last_serial = Column(Integer, nullable=False, server_default=sql.text("0")) allow_legacy_files = Column( Boolean, nullable=False, server_default=sql.false(), ) users = orm.relationship( User, secondary=Role.__table__, backref="projects", ) releases = orm.relationship( "Release", backref="project", cascade="all, delete-orphan", order_by=lambda: Release._pypi_ordering.desc(), ) def __getitem__(self, version): session = orm.object_session(self) try: return (session.query(Release).filter((Release.project == self) & ( Release.version == version)).one()) except NoResultFound: raise KeyError from None def __acl__(self): session = orm.object_session(self) acls = [] # Get all of the users for this project. query = session.query(Role).filter(Role.project == self) query = query.options(orm.lazyload("project")) query = query.options(orm.joinedload("user").lazyload("emails")) for role in sorted( query.all(), key=lambda x: ["Owner", "Maintainer"].index(x.role_name)): acls.append((Allow, role.user.id, ["upload"])) return acls @property def documentation_url(self): # TODO: Move this into the database and elimnate the use of the # threadlocal here. request = get_current_request() # If the project doesn't have docs, then we'll just return a None here. if not self.has_docs: return return request.route_url("legacy.docs", project=self.name)
def file_upload(request): # Before we do anything, if there isn't an authenticated user with this # request, then we'll go ahead and bomb out. if request.authenticated_userid is None: raise _exc_with_message( HTTPForbidden, "Invalid or non-existent authentication information.", ) # distutils "helpfully" substitutes unknown, but "required" values with the # string "UNKNOWN". This is basically never what anyone actually wants so # we'll just go ahead and delete anything whose value is UNKNOWN. for key in list(request.POST): if request.POST.get(key) == "UNKNOWN": del request.POST[key] # We require protocol_version 1, it's the only supported version however # passing a different version should raise an error. if request.POST.get("protocol_version", "1") != "1": raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.") # Check if any fields were supplied as a tuple and have become a # FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a # FieldStorage, however. # ref: https://github.com/pypa/warehouse/issues/2185 # ref: https://github.com/pypa/warehouse/issues/2491 for field in set(request.POST) - {'content', 'gpg_signature'}: values = request.POST.getall(field) if any(isinstance(value, FieldStorage) for value in values): raise _exc_with_message( HTTPBadRequest, f"{field}: Should not be a tuple.", ) # Look up all of the valid classifiers all_classifiers = request.db.query(Classifier).all() # Validate and process the incoming metadata. form = MetadataForm(request.POST) form.classifiers.choices = [(c.classifier, c.classifier) for c in all_classifiers] if not form.validate(): for field_name in _error_message_order: if field_name in form.errors: break else: field_name = sorted(form.errors.keys())[0] if field_name in form: if form[field_name].description: error_message = ( "{value!r} is an invalid value for {field}. ".format( value=form[field_name].data, field=form[field_name].description) + "Error: {} ".format(form.errors[field_name][0]) + "see " "https://packaging.python.org/specifications/core-metadata" ) else: error_message = "{field}: {msgs[0]}".format( field=field_name, msgs=form.errors[field_name], ) else: error_message = "Error: {}".format(form.errors[field_name][0]) raise _exc_with_message( HTTPBadRequest, error_message, ) # Ensure that we have file data in the request. if "content" not in request.POST: raise _exc_with_message( HTTPBadRequest, "Upload payload does not have a file.", ) # Look up the project first before doing anything else, this is so we can # automatically register it if we need to and can check permissions before # going any further. try: project = (request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name( form.name.data)).one()) except NoResultFound: # Check for AdminFlag set by a PyPI Administrator disabling new project # registration, reasons for this include Spammers, security # vulnerabilities, or just wanting to be lazy and not worry ;) if AdminFlag.is_enabled(request.db, 'disallow-new-project-registration'): raise _exc_with_message( HTTPForbidden, ("New Project Registration Temporarily Disabled " "See https://pypi.org/help#admin-intervention for details"), ) from None # Ensure that user has at least one verified email address. This should # reduce the ease of spam account creation and activity. # TODO: Once legacy is shutdown consider the condition here, perhaps # move to user.is_active or some other boolean if not any(email.verified for email in request.user.emails): raise _exc_with_message( HTTPBadRequest, ("User {!r} has no verified email addresses, please verify " "at least one address before registering a new project on " "PyPI. See https://pypi.org/help/#verified-email " "for more information.").format(request.user.username), ) from None # Before we create the project, we're going to check our blacklist to # see if this project is even allowed to be registered. If it is not, # then we're going to deny the request to create this project. if request.db.query(exists().where( BlacklistedProject.name == func.normalize_pep426_name( form.name.data))).scalar(): raise _exc_with_message( HTTPBadRequest, ("The name {!r} is not allowed. " "See https://pypi.org/help/#project-name " "for more information.").format(form.name.data), ) from None # Also check for collisions with Python Standard Library modules. if (packaging.utils.canonicalize_name(form.name.data) in STDLIB_PROHIBITTED): raise _exc_with_message( HTTPBadRequest, ("The name {!r} is not allowed (conflict with Python " "Standard Library module name). See " "https://pypi.org/help/#project-name for more information." ).format(form.name.data), ) from None # The project doesn't exist in our database, so we'll add it along with # a role setting the current user as the "Owner" of the project. project = Project(name=form.name.data) request.db.add(project) request.db.add( Role(user=request.user, project=project, role_name="Owner")) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=project.name, action="create", submitted_by=request.user, submitted_from=request.remote_addr, ), ) request.db.add( JournalEntry( name=project.name, action="add Owner {}".format(request.user.username), submitted_by=request.user, submitted_from=request.remote_addr, ), ) # Check that the user has permission to do things to this project, if this # is a new project this will act as a sanity check for the role we just # added above. if not request.has_permission("upload", project): raise _exc_with_message( HTTPForbidden, ("The user '{0}' is not allowed to upload to project '{1}'. " "See https://pypi.org/help#project-name for more information." ).format(request.user.username, project.name)) try: canonical_version = packaging.utils.canonicalize_version( form.version.data) release = (request.db.query(Release).filter( (Release.project == project) & (Release.canonical_version == canonical_version)).one()) except MultipleResultsFound: # There are multiple releases of this project which have the same # canonical version that were uploaded before we checked for # canonical version equivalence, so return the exact match instead release = (request.db.query( Release).filter((Release.project == project) & (Release.version == form.version.data)).one()) except NoResultFound: release = Release( project=project, _classifiers=[ c for c in all_classifiers if c.classifier in form.classifiers.data ], _pypi_hidden=False, dependencies=list( _construct_dependencies( form, { "requires": DependencyKind.requires, "provides": DependencyKind.provides, "obsoletes": DependencyKind.obsoletes, "requires_dist": DependencyKind.requires_dist, "provides_dist": DependencyKind.provides_dist, "obsoletes_dist": DependencyKind.obsoletes_dist, "requires_external": DependencyKind.requires_external, "project_urls": DependencyKind.project_url, })), canonical_version=canonical_version, **{ k: getattr(form, k).data for k in { # This is a list of all the fields in the form that we # should pull off and insert into our new release. "version", "summary", "description", "license", "author", "author_email", "maintainer", "maintainer_email", "keywords", "platform", "home_page", "download_url", "requires_python", } }) request.db.add(release) # TODO: This should be handled by some sort of database trigger or # a SQLAlchemy hook or the like instead of doing it inline in # this view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="new release", submitted_by=request.user, submitted_from=request.remote_addr, ), ) # TODO: We need a better solution to this than to just do it inline inside # this method. Ideally the version field would just be sortable, but # at least this should be some sort of hook or trigger. releases = (request.db.query(Release).filter( Release.project == project).all()) for i, r in enumerate( sorted(releases, key=lambda x: packaging.version.parse(x.version))): r._pypi_ordering = i # TODO: Again, we should figure out a better solution to doing this than # just inlining this inside this method. if project.autohide: for r in releases: r._pypi_hidden = bool(not r == release) # Pull the filename out of our POST data. filename = request.POST["content"].filename # Make sure that the filename does not contain any path separators. if "/" in filename or "\\" in filename: raise _exc_with_message( HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name.", ) # Make sure the filename ends with an allowed extension. if _dist_file_regexes[project.allow_legacy_files].search(filename) is None: raise _exc_with_message( HTTPBadRequest, "Invalid file extension. PEP 527 requires one of: .egg, .tar.gz, " ".whl, .zip (https://www.python.org/dev/peps/pep-0527/).") # Make sure that our filename matches the project that it is being uploaded # to. prefix = pkg_resources.safe_name(project.name).lower() if not pkg_resources.safe_name(filename).lower().startswith(prefix): raise _exc_with_message( HTTPBadRequest, "The filename for {!r} must start with {!r}.".format( project.name, prefix, )) # Check the content type of what is being uploaded if (not request.POST["content"].type or request.POST["content"].type.startswith("image/")): raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.") # Ensure that the package filetpye is allowed. # TODO: Once PEP 527 is completely implemented we should be able to delete # this and just move it into the form itself. if (not project.allow_legacy_files and form.filetype.data not in {"sdist", "bdist_wheel", "bdist_egg"}): raise _exc_with_message(HTTPBadRequest, "Unknown type of file.") # The project may or may not have a file size specified on the project, if # it does then it may or may not be smaller or larger than our global file # size limits. file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit])) with tempfile.TemporaryDirectory() as tmpdir: temporary_filename = os.path.join(tmpdir, filename) # Buffer the entire file onto disk, checking the hash of the file as we # go along. with open(temporary_filename, "wb") as fp: file_size = 0 file_hashes = { "md5": hashlib.md5(), "sha256": hashlib.sha256(), "blake2_256": hashlib.blake2b(digest_size=256 // 8), } for chunk in iter(lambda: request.POST["content"].file.read(8096), b""): file_size += len(chunk) if file_size > file_size_limit: raise _exc_with_message( HTTPBadRequest, "File too large. " + "Limit for project {name!r} is {limit}MB".format( name=project.name, limit=file_size_limit // (1024 * 1024), )) fp.write(chunk) for hasher in file_hashes.values(): hasher.update(chunk) # Take our hash functions and compute the final hashes for them now. file_hashes = { k: h.hexdigest().lower() for k, h in file_hashes.items() } # Actually verify the digests that we've gotten. We're going to use # hmac.compare_digest even though we probably don't actually need to # because it's better safe than sorry. In the case of multiple digests # we expect them all to be given. if not all([ hmac.compare_digest( getattr(form, "{}_digest".format(digest_name)).data.lower(), digest_value, ) for digest_name, digest_value in file_hashes.items() if getattr(form, "{}_digest".format(digest_name)).data ]): raise _exc_with_message( HTTPBadRequest, "The digest supplied does not match a digest calculated " "from the uploaded file.") # Check to see if the file that was uploaded exists already or not. is_duplicate = _is_duplicate_file(request.db, filename, file_hashes) if is_duplicate: return Response() elif is_duplicate is not None: raise _exc_with_message( HTTPBadRequest, "File already exists. " "See " + request.route_url('help', _anchor='file-name-reuse')) # Check to see if the file that was uploaded exists in our filename log if (request.db.query( request.db.query(Filename).filter( Filename.filename == filename).exists()).scalar()): raise _exc_with_message( HTTPBadRequest, "This filename has previously been used, you should use a " "different version. " "See " + request.route_url('help', _anchor='file-name-reuse'), ) # Check to see if uploading this file would create a duplicate sdist # for the current release. if (form.filetype.data == "sdist" and request.db.query( request.db.query(File).filter((File.release == release) & ( File.packagetype == "sdist")).exists()).scalar()): raise _exc_with_message( HTTPBadRequest, "Only one sdist may be uploaded per release.", ) # Check the file to make sure it is a valid distribution file. if not _is_valid_dist_file(temporary_filename, form.filetype.data): raise _exc_with_message( HTTPBadRequest, "Invalid distribution file.", ) # Check that if it's a binary wheel, it's on a supported platform if filename.endswith(".whl"): wheel_info = _wheel_file_re.match(filename) plats = wheel_info.group("plat").split(".") for plat in plats: if not _valid_platform_tag(plat): raise _exc_with_message( HTTPBadRequest, "Binary wheel '{filename}' has an unsupported " "platform tag '{plat}'.".format(filename=filename, plat=plat)) # Also buffer the entire signature file to disk. if "gpg_signature" in request.POST: has_signature = True with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp: signature_size = 0 for chunk in iter( lambda: request.POST["gpg_signature"].file.read(8096), b""): signature_size += len(chunk) if signature_size > MAX_SIGSIZE: raise _exc_with_message( HTTPBadRequest, "Signature too large.", ) fp.write(chunk) # Check whether signature is ASCII armored with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp: if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"): raise _exc_with_message( HTTPBadRequest, "PGP signature is not ASCII armored.", ) else: has_signature = False # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add(Filename(filename=filename)) # Store the information about the file in the database. file_ = File( release=release, filename=filename, python_version=form.pyversion.data, packagetype=form.filetype.data, comment_text=form.comment.data, size=file_size, has_signature=bool(has_signature), md5_digest=file_hashes["md5"], sha256_digest=file_hashes["sha256"], blake2_256_digest=file_hashes["blake2_256"], # Figure out what our filepath is going to be, we're going to use a # directory structure based on the hash of the file contents. This # will ensure that the contents of the file cannot change without # it also changing the path that the file is saved too. path="/".join([ file_hashes[PATH_HASHER][:2], file_hashes[PATH_HASHER][2:4], file_hashes[PATH_HASHER][4:], filename, ]), ) request.db.add(file_) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="add {python_version} file {filename}".format( python_version=file_.python_version, filename=file_.filename, ), submitted_by=request.user, submitted_from=request.remote_addr, ), ) # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is # committed. storage = request.find_service(IFileStorage) storage.store( file_.path, os.path.join(tmpdir, filename), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) if has_signature: storage.store( file_.pgp_path, os.path.join(tmpdir, filename + ".asc"), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) # TODO: Once we no longer have the legacy code base running PyPI we can # go ahead and delete this tiny bit of shim code, since it only # exists to purge stuff on legacy PyPI when uploaded to Warehouse old_domain = request.registry.settings.get("warehouse.legacy_domain") if old_domain: request.tm.get().addAfterCommitHook( _legacy_purge, args=["https://{}/pypi".format(old_domain)], kws={"data": { ":action": "purge", "project": project.name }}, ) return Response()
def release_data(request, package_name, version): try: release = (request.db.query(Release).join(Project).filter(( Project.normalized_name == func.normalize_pep426_name(package_name) ) & (Release.version == version)).one()) except NoResultFound: return {} stats_svc = request.find_service(IDownloadStatService) return { "name": release.project.name, "version": release.version, "stable_version": release.project.stable_version, "bugtrack_url": release.project.bugtrack_url, "package_url": request.route_url( "packaging.project", name=release.project.name, ), "release_url": request.route_url( "packaging.release", name=release.project.name, version=release.version, ), "docs_url": release.project.documentation_url, "home_page": release.home_page, "download_url": release.download_url, "project_url": list(release.project_urls), "author": release.author, "author_email": release.author_email, "maintainer": release.maintainer, "maintainer_email": release.maintainer_email, "summary": release.summary, "description": release.description, "license": release.license, "keywords": release.keywords, "platform": release.platform, "classifiers": list(release.classifiers), "requires": list(release.requires), "requires_dist": list(release.requires_dist), "provides": list(release.provides), "provides_dist": list(release.provides_dist), "obsoletes": list(release.obsoletes), "obsoletes_dist": list(release.obsoletes_dist), "requires_python": release.requires_python, "requires_external": list(release.requires_external), "_pypi_ordering": release._pypi_ordering, "_pypi_hidden": release._pypi_hidden, "downloads": { "last_day": stats_svc.get_daily_stats(release.project.name), "last_week": stats_svc.get_weekly_stats(release.project.name), "last_month": stats_svc.get_monthly_stats(release.project.name), }, }
class Project(db.Model, Syncable): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.Text, nullable=False, index=True) normalized_name = orm.column_property(func.normalize_pep426_name(name)) description = db.Column(db.Text) html_url = db.Column(db.String(255)) subscribers_count = db.Column(db.SmallInteger, default=0, nullable=False) stargazers_count = db.Column(db.SmallInteger, default=0, nullable=False) forks_count = db.Column(db.SmallInteger, default=0, nullable=False) open_issues_count = db.Column(db.SmallInteger, default=0, nullable=False) is_active = db.Column(db.Boolean, default=True, nullable=False, index=True) transfer_issue_url = db.Column(db.String(255)) membership = db.relationship("ProjectMembership", backref="project", lazy="dynamic") credentials = db.relationship("ProjectCredential", backref="project", lazy="dynamic") uploads = db.relationship( "ProjectUpload", backref="project", lazy="dynamic", order_by=lambda: ProjectUpload.ordering.desc().nullslast(), ) created_at = db.Column(db.DateTime, nullable=True) updated_at = db.Column(db.DateTime, nullable=True) pushed_at = db.Column(db.DateTime, nullable=True) __tablename__ = "projects" __table_args__ = ( db.Index("release_name_idx", "name"), db.Index("release_name_is_active_idx", "name", "is_active"), ) def __str__(self): return self.name @aggregated("uploads", db.Column(db.SmallInteger)) def uploads_count(self): return db.func.count("1") @property def current_user_is_member(self): if not current_user: return False elif not current_user.is_authenticated: return False elif current_user_is_roadie(): return True else: return current_user.id in self.member_ids @property def member_ids(self): return [member.user.id for member in self.membership.all()] @property def leads(self): leads = self.membership.filter( ProjectMembership.is_lead.is_(True), ProjectMembership.user_id.in_(User.active_members().options( orm.load_only("id"))), ) return [member.user for member in leads] @property def pypi_json_url(self): return f"https://pypi.org/pypi/{self.normalized_name}/json" # noqa
def file_upload(request): # Before we do anything, if there isn't an authenticated user with this # request, then we'll go ahead and bomb out. if request.authenticated_userid is None: raise _exc_with_message( HTTPForbidden, "Invalid or non-existent authentication information.", ) # distutils "helpfully" substitutes unknown, but "required" values with the # string "UNKNOWN". This is basically never what anyone actually wants so # we'll just go ahead and delete anything whose value is UNKNOWN. for key in list(request.POST): if request.POST.get(key) == "UNKNOWN": del request.POST[key] # We require protocol_version 1, it's the only supported version however # passing a different version should raise an error. if request.POST.get("protocol_version", "1") != "1": raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.") # Look up all of the valid classifiers all_classifiers = request.db.query(Classifier).all() # Validate and process the incoming metadata. form = MetadataForm(request.POST) form.classifiers.choices = [ (c.classifier, c.classifier) for c in all_classifiers ] if not form.validate(): for field_name in _error_message_order: if field_name in form.errors: break else: field_name = sorted(form.errors.keys())[0] raise _exc_with_message( HTTPBadRequest, "{field}: {msgs[0]}".format( field=field_name, msgs=form.errors[field_name], ), ) # TODO: We need a better method of blocking names rather than just # hardcoding some names into source control. if form.name.data.lower() in {"requirements.txt", "rrequirements.txt"}: raise _exc_with_message( HTTPBadRequest, "The name {!r} is not allowed.".format(form.name.data), ) # Ensure that we have file data in the request. if "content" not in request.POST: raise _exc_with_message( HTTPBadRequest, "Upload payload does not have a file.", ) # Look up the project first before doing anything else, this is so we can # automatically register it if we need to and can check permissions before # going any further. try: project = ( request.db.query(Project) .filter( Project.normalized_name == func.normalize_pep426_name(form.name.data)).one() ) except NoResultFound: # The project doesn't exist in our database, so we'll add it along with # a role setting the current user as the "Owner" of the project. project = Project(name=form.name.data) request.db.add(project) request.db.add( Role(user=request.user, project=project, role_name="Owner") ) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=project.name, action="create", submitted_by=request.user, submitted_from=request.client_addr, ), ) request.db.add( JournalEntry( name=project.name, action="add Owner {}".format(request.user.username), submitted_by=request.user, submitted_from=request.client_addr, ), ) # Check that the user has permission to do things to this project, if this # is a new project this will act as a sanity check for the role we just # added above. if not request.has_permission("upload", project): raise _exc_with_message( HTTPForbidden, "You are not allowed to upload to {!r}.".format(project.name) ) try: release = ( request.db.query(Release) .filter( (Release.project == project) & (Release.version == form.version.data)).one() ) except NoResultFound: release = Release( project=project, _classifiers=[ c for c in all_classifiers if c.classifier in form.classifiers.data ], _pypi_hidden=False, dependencies=list(_construct_dependencies( form, { "requires": DependencyKind.requires, "provides": DependencyKind.provides, "obsoletes": DependencyKind.obsoletes, "requires_dist": DependencyKind.requires_dist, "provides_dist": DependencyKind.provides_dist, "obsoletes_dist": DependencyKind.obsoletes_dist, "requires_external": DependencyKind.requires_external, "project_urls": DependencyKind.project_url, } )), **{ k: getattr(form, k).data for k in { # This is a list of all the fields in the form that we # should pull off and insert into our new release. "version", "summary", "description", "license", "author", "author_email", "maintainer", "maintainer_email", "keywords", "platform", "home_page", "download_url", "requires_python", } } ) request.db.add(release) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="new release", submitted_by=request.user, submitted_from=request.client_addr, ), ) # TODO: We need a better solution to this than to just do it inline inside # this method. Ideally the version field would just be sortable, but # at least this should be some sort of hook or trigger. releases = ( request.db.query(Release) .filter(Release.project == project) .all() ) for i, r in enumerate(sorted( releases, key=lambda x: packaging.version.parse(x.version))): r._pypi_ordering = i # TODO: Again, we should figure out a better solution to doing this than # just inlining this inside this method. if project.autohide: for r in releases: r._pypi_hidden = bool(not r == release) # Pull the filename out of our POST data. filename = request.POST["content"].filename # Make sure that the filename does not contain any path separators. if "/" in filename or "\\" in filename: raise _exc_with_message( HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name.", ) # Make sure the filename ends with an allowed extension. if _dist_file_re.search(filename) is None: raise _exc_with_message(HTTPBadRequest, "Invalid file extension.") # Make sure that our filename matches the project that it is being uploaded # to. prefix = pkg_resources.safe_name(project.name).lower() if not pkg_resources.safe_name(filename).lower().startswith(prefix): raise _exc_with_message( HTTPBadRequest, "The filename for {!r} must start with {!r}.".format( project.name, prefix, ) ) # Check the content type of what is being uploaded if (not request.POST["content"].type or request.POST["content"].type.startswith("image/")): raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.") # Check to see if the file that was uploaded exists already or not. if request.db.query( request.db.query(File) .filter(File.filename == filename) .exists()).scalar(): raise _exc_with_message(HTTPBadRequest, "File already exists.") # Check to see if the file that was uploaded exists in our filename log. if (request.db.query( request.db.query(Filename) .filter(Filename.filename == filename) .exists()).scalar()): raise _exc_with_message( HTTPBadRequest, "This filename has previously been used, you should use a " "different version.", ) # The project may or may not have a file size specified on the project, if # it does then it may or may not be smaller or larger than our global file # size limits. file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit])) with tempfile.TemporaryDirectory() as tmpdir: temporary_filename = os.path.join(tmpdir, filename) # Buffer the entire file onto disk, checking the hash of the file as we # go along. with open(temporary_filename, "wb") as fp: file_size = 0 file_hashes = { "md5": hashlib.md5(), "sha256": hashlib.sha256(), "blake2_256": blake2b(digest_size=256 // 8), } for chunk in iter( lambda: request.POST["content"].file.read(8096), b""): file_size += len(chunk) if file_size > file_size_limit: raise _exc_with_message(HTTPBadRequest, "File too large.") fp.write(chunk) for hasher in file_hashes.values(): hasher.update(chunk) # Take our hash functions and compute the final hashes for them now. file_hashes = { k: h.hexdigest().lower() for k, h in file_hashes.items() } # Actually verify the digests that we've gotten. We're going to use # hmac.compare_digest even though we probably don't actually need to # because it's better safe than sorry. In the case of multiple digests # we expect them all to be given. if not all([ hmac.compare_digest( getattr(form, "{}_digest".format(digest_name)).data.lower(), digest_value, ) for digest_name, digest_value in file_hashes.items() if getattr(form, "{}_digest".format(digest_name)).data ]): raise _exc_with_message( HTTPBadRequest, "The digest supplied does not match a digest calculated " "from the uploaded file." ) # Check the file to make sure it is a valid distribution file. if not _is_valid_dist_file(temporary_filename, form.filetype.data): raise _exc_with_message( HTTPBadRequest, "Invalid distribution file.", ) # Check that if it's a binary wheel, it's on a supported platform if filename.endswith(".whl"): wheel_info = _wheel_file_re.match(filename) plats = wheel_info.group("plat").split(".") for plat in plats: if not _valid_platform_tag(plat): raise _exc_with_message( HTTPBadRequest, "Binary wheel '{filename}' has an unsupported " "platform tag '{plat}'." .format(filename=filename, plat=plat) ) # Also buffer the entire signature file to disk. if "gpg_signature" in request.POST: has_signature = True with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp: signature_size = 0 for chunk in iter( lambda: request.POST["gpg_signature"].file.read(8096), b""): signature_size += len(chunk) if signature_size > MAX_SIGSIZE: raise _exc_with_message( HTTPBadRequest, "Signature too large.", ) fp.write(chunk) # Check whether signature is ASCII armored with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp: if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"): raise _exc_with_message( HTTPBadRequest, "PGP signature is not ASCII armored.", ) else: has_signature = False # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add(Filename(filename=filename)) # Store the information about the file in the database. file_ = File( release=release, filename=filename, python_version=form.pyversion.data, packagetype=form.filetype.data, comment_text=form.comment.data, size=file_size, has_signature=bool(has_signature), md5_digest=file_hashes["md5"], sha256_digest=file_hashes["sha256"], blake2_256_digest=file_hashes["blake2_256"], # Figure out what our filepath is going to be, we're going to use a # directory structure based on the hash of the file contents. This # will ensure that the contents of the file cannot change without # it also changing the path that the file is saved too. path="/".join([ file_hashes[PATH_HASHER][:2], file_hashes[PATH_HASHER][2:4], file_hashes[PATH_HASHER][4:], filename, ]), ) request.db.add(file_) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="add {python_version} file {filename}".format( python_version=file_.python_version, filename=file_.filename, ), submitted_by=request.user, submitted_from=request.client_addr, ), ) # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is # committed. storage = request.find_service(IFileStorage) storage.store( file_.path, os.path.join(tmpdir, filename), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) if has_signature: storage.store( file_.pgp_path, os.path.join(tmpdir, filename + ".asc"), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) # TODO: Once we no longer have the legacy code base running PyPI we can # go ahead and delete this tiny bit of shim code, since it only # exists to purge stuff on legacy PyPI when uploaded to Warehouse old_domain = request.registry.settings.get("warehouse.legacy_domain") if old_domain: request.tm.get().addAfterCommitHook( _legacy_purge, args=["https://{}/pypi".format(old_domain)], kws={"data": {":action": "purge", "project": project.name}}, ) return Response()
class Project(SitemapMixin, db.Model): __tablename__ = "projects" __table_args__ = (CheckConstraint( "name ~* '^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$'::text", name="projects_valid_name", ), ) __repr__ = make_repr("name") name = Column(Text, nullable=False) normalized_name = orm.column_property(func.normalize_pep426_name(name)) created = Column( DateTime(timezone=False), nullable=False, server_default=sql.func.now(), index=True, ) has_docs = Column(Boolean) upload_limit = Column(Integer, nullable=True) last_serial = Column(Integer, nullable=False, server_default=sql.text("0")) zscore = Column(Float, nullable=True) total_size = Column(BigInteger, server_default=sql.text("0")) users = orm.relationship(User, secondary=Role.__table__, backref="projects") releases = orm.relationship( "Release", backref="project", cascade="all, delete-orphan", order_by=lambda: Release._pypi_ordering.desc(), passive_deletes=True, ) events = orm.relationship("ProjectEvent", backref="project", cascade="all, delete-orphan", lazy=True) def __getitem__(self, version): session = orm.object_session(self) canonical_version = packaging.utils.canonicalize_version(version) try: return (session.query(Release).filter( Release.project == self, Release.canonical_version == canonical_version, ).one()) except MultipleResultsFound: # There are multiple releases of this project which have the same # canonical version that were uploaded before we checked for # canonical version equivalence, so return the exact match instead try: return (session.query(Release).filter( Release.project == self, Release.version == version).one()) except NoResultFound: # There are multiple releases of this project which have the # same canonical version, but none that have the exact version # specified, so just 404 raise KeyError from None except NoResultFound: raise KeyError from None def __acl__(self): session = orm.object_session(self) acls = [ (Allow, "group:admins", "admin"), (Allow, "group:moderators", "moderator"), ] # Get all of the users for this project. query = session.query(Role).filter(Role.project == self) query = query.options(orm.lazyload("project")) query = query.options(orm.joinedload("user").lazyload("emails")) query = query.join(User).order_by(User.id.asc()) for role in sorted( query.all(), key=lambda x: ["Owner", "Maintainer"].index(x.role_name)): if role.role_name == "Owner": acls.append( (Allow, str(role.user.id), ["manage:project", "upload"])) else: acls.append((Allow, str(role.user.id), ["upload"])) return acls def record_event(self, *, tag, ip_address, additional=None): session = orm.object_session(self) event = ProjectEvent(project=self, tag=tag, ip_address=ip_address, additional=additional) session.add(event) session.flush() return event @property def documentation_url(self): # TODO: Move this into the database and eliminate the use of the # threadlocal here. request = get_current_request() # If the project doesn't have docs, then we'll just return a None here. if not self.has_docs: return return request.route_url("legacy.docs", project=self.name) @property def all_versions(self): return (orm.object_session(self).query( Release.version, Release.created, Release.is_prerelease, Release.yanked).filter(Release.project == self).order_by( Release._pypi_ordering.desc()).all()) @property def latest_version(self): return (orm.object_session(self).query( Release.version, Release.created, Release.is_prerelease).filter( Release.project == self, Release.yanked.is_(False)).order_by( Release.is_prerelease.nullslast(), Release._pypi_ordering.desc()).first())
def add_blacklist(request): project_name = request.POST.get("project") if project_name is None: raise HTTPBadRequest("Must have a project to confirm.") comment = request.POST.get("comment", "") # Verify that the user has confirmed the request to blacklist. confirm = request.POST.get("confirm") if not confirm: request.session.flash( "Must confirm the blacklist request.", queue="error", ) return HTTPSeeOther(request.current_route_path()) elif canonicalize_name(confirm) != canonicalize_name(project_name): request.session.flash( f"{confirm!r} is not the same as {project_name!r}", queue="error", ) return HTTPSeeOther(request.current_route_path()) # Add our requested blacklist. request.db.add( BlacklistedProject( name=project_name, comment=comment, blacklisted_by=request.user, )) # Go through and delete anything that we need to delete so that our # blacklist actually blocks things and isn't ignored (since the blacklist # only takes effect on new project registration). # TODO: This should be in a generic function somewhere instead of putting # it here, however this will have to do for now. # TODO: We don't actually delete files from the data store. We should add # some kind of garbage collection at some point. project = (request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name( project_name)).first()) if project is not None: request.db.add( JournalEntry( name=project.name, action="remove", submitted_by=request.user, submitted_from=request.remote_addr, )) request.db.query(Role).filter(Role.project == project).delete() request.db.query(File).filter(File.name == project.name).delete() (request.db.query(Dependency).filter( Dependency.name == project.name).delete()) (request.db.execute(release_classifiers.delete().where( release_classifiers.c.name == project.name))) request.db.query(Release).filter(Release.name == project.name).delete() request.db.query(Project).filter(Project.name == project.name).delete() request.session.flash( f"Successfully blacklisted {project_name!r}", queue="success", ) return HTTPSeeOther(request.route_path("admin.blacklist.list"))
class Organization(HasEvents, db.Model): __tablename__ = "organizations" __table_args__ = (CheckConstraint( "name ~* '^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$'::text", name="organizations_valid_name", ), ) __repr__ = make_repr("name") name = Column(Text, nullable=False) normalized_name = orm.column_property(func.normalize_pep426_name(name)) display_name = Column(Text, nullable=False) orgtype = Column( Enum(OrganizationType, values_callable=lambda x: [e.value for e in x]), nullable=False, ) link_url = Column(URLType, nullable=False) description = Column(Text, nullable=False) is_active = Column(Boolean, nullable=False, server_default=sql.false()) is_approved = Column(Boolean) created = Column( DateTime(timezone=False), nullable=False, server_default=sql.func.now(), index=True, ) date_approved = Column( DateTime(timezone=False), nullable=True, onupdate=func.now(), ) users = orm.relationship( User, secondary=OrganizationRole.__table__, backref="organizations" # type: ignore # noqa ) projects = orm.relationship( "Project", secondary=OrganizationProject.__table__, backref="organizations" # type: ignore # noqa ) def record_event(self, *, tag, ip_address, additional={}): """Record organization name in events in case organization is ever deleted.""" super().record_event( tag=tag, ip_address=ip_address, additional={ "organization_name": self.name, **additional }, ) def __acl__(self): session = orm.object_session(self) acls = [ (Allow, "group:admins", "admin"), (Allow, "group:moderators", "moderator"), ] # Get all of the users for this organization. query = session.query(OrganizationRole).filter( OrganizationRole.organization == self) query = query.options(orm.lazyload("organization")) query = query.join(User).order_by(User.id.asc()) for role in sorted( query.all(), key=lambda x: [e.value for e in OrganizationRoleType].index( x.role_name), ): # Allow all people in organization read access. # Allow write access depending on role. if role.role_name == OrganizationRoleType.Owner: acls.append(( Allow, f"user:{role.user.id}", ["view:organization", "manage:organization"], )) elif role.role_name == OrganizationRoleType.BillingManager: acls.append(( Allow, f"user:{role.user.id}", ["view:organization", "manage:billing"], )) elif role.role_name == OrganizationRoleType.Manager: acls.append(( Allow, f"user:{role.user.id}", ["view:organization", "manage:team"], )) else: # No member-specific write access needed for now. acls.append( (Allow, f"user:{role.user.id}", ["view:organization"])) return acls
def file_upload(request): # Before we do anything, if their isn't an authenticated user with this # request, then we'll go ahead and bomb out. if request.authenticated_userid is None: raise _exc_with_message(HTTPForbidden, "Invalid or non-existent authentication information.") # distutils "helpfully" substitutes unknown, but "required" values with the # string "UNKNOWN". This is basically never what anyone actually wants so # we'll just go ahead and delete anything whose value is UNKNOWN. for key in list(request.POST): if request.POST.get(key) == "UNKNOWN": del request.POST[key] # We require protocol_version 1, it's the only supported version however # passing a different version should raise an error. if request.POST.get("protocol_version", "1") != "1": raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.") # Look up all of the valid classifiers all_classifiers = request.db.query(Classifier).all() # Validate and process the incoming metadata. form = MetadataForm(request.POST) form.classifiers.choices = [(c.classifier, c.classifier) for c in all_classifiers] if not form.validate(): for field_name in _error_message_order: if field_name in form.errors: break else: field_name = sorted(form.errors.keys())[0] raise _exc_with_message( HTTPBadRequest, "{field}: {msgs[0]}".format(field=field_name, msgs=form.errors[field_name]) ) # TODO: We need a better method of blocking names rather than jsut # hardcoding some names into source control. if form.name.data.lower() in {"requirements.txt", "rrequirements.txt"}: raise _exc_with_message(HTTPBadRequest, "The name {!r} is not allowed.".format(form.name.data)) # Ensure that we have file data in the request. if "content" not in request.POST: raise _exc_with_message(HTTPBadRequest, "Upload payload does not have a file.") # Look up the project first before doing anything else, this is so we can # automatically register it if we need to and can check permissions before # going any further. try: project = ( request.db.query(Project) .filter(Project.normalized_name == func.normalize_pep426_name(form.name.data)) .one() ) except NoResultFound: # The project doesn't exist in our database, so we'll add it along with # a role setting the current user as the "Owner" of the project. project = Project(name=form.name.data) request.db.add(project) request.db.add(Role(user=request.user, project=project, role_name="Owner")) # Check that the user has permission to do things to this project, if this # is a new project this will act as a sanity check for the role we just # added above. if not request.has_permission("upload", project): raise _exc_with_message(HTTPForbidden, "You are not allowed to upload to {!r}.".format(project.name)) try: release = ( request.db.query(Release) .filter((Release.project == project) & (Release.version == form.version.data)) .one() ) except NoResultFound: release = Release( project=project, _classifiers=[c for c in all_classifiers if c.classifier in form.classifiers.data], dependencies=list( _construct_dependencies( form, { "requires": DependencyKind.requires, "provides": DependencyKind.provides, "obsoletes": DependencyKind.obsoletes, "requires_dist": DependencyKind.requires_dist, "provides_dist": DependencyKind.provides_dist, "obsoletes_dist": DependencyKind.obsoletes_dist, "requires_external": DependencyKind.requires_external, "project_urls": DependencyKind.project_url, }, ) ), **{ k: getattr(form, k).data for k in { # This is a list of all the fields in the form that we # should pull off and insert into our new release. "version", "summary", "description", "license", "author", "author_email", "maintainer", "maintainer_email", "keywords", "platform", "home_page", "download_url", "requires_python", } } ) request.db.add(release) # TODO: We need a better solution to this than to just do it inline inside # this method. Ideally the version field would just be sortable, but # at least this should be some sort of hook or trigger. releases = request.db.query(Release).filter(Release.project == project).all() for i, r in enumerate(sorted(releases, key=lambda x: packaging.version.parse(x.version))): r._pypi_ordering = i # Pull the filename out of our POST data. filename = request.POST["content"].filename # Make sure that the filename does not contain and path seperators. if "/" in filename or "\\" in filename: raise _exc_with_message(HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name.") # Make sure the filename ends with an allowed extension. if _dist_file_re.search(filename) is None: raise _exc_with_message(HTTPBadRequest, "Invalid file extension.") # Make sure that our filename matches the project that it is being uploaded # to. prefix = pkg_resources.safe_name(project.name).lower() if not pkg_resources.safe_name(filename).lower().startswith(prefix): raise _exc_with_message( HTTPBadRequest, "The filename for {!r} must start with {!r}.".format(project.name, prefix) ) # Check to see if the file that was uploaded exists already or not. if request.db.query(request.db.query(File).filter(File.filename == filename).exists()).scalar(): raise _exc_with_message(HTTPBadRequest, "File already exists.") # Check to see if the file that was uploaded exists in our filename log. if request.db.query(request.db.query(Filename).filter(Filename.filename == filename).exists()).scalar(): raise _exc_with_message( HTTPBadRequest, "This filename has previously been used, you should use a " "different version." ) # The project may or may not have a file size specified on the project, if # it does then it may or may not be smaller or larger than our global file # size limits. file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit])) with tempfile.TemporaryDirectory() as tmpdir: # Buffer the entire file onto disk, checking the hash of the file as we # go along. with open(os.path.join(tmpdir, filename), "wb") as fp: file_size = 0 file_hash = hashlib.md5() for chunk in iter(lambda: request.POST["content"].file.read(8096), b""): file_size += len(chunk) if file_size > file_size_limit: raise _exc_with_message(HTTPBadRequest, "File too large.") fp.write(chunk) file_hash.update(chunk) # Actually verify that the md5 hash of the file matches the expected # md5 hash. We probably don't actually need to use hmac.compare_digest # here since both the md5_digest and the file whose file_hash we've # computed comes from the remote user, however better safe than sorry. if not hmac.compare_digest(form.md5_digest.data, file_hash.hexdigest()): raise _exc_with_message( HTTPBadRequest, "The MD5 digest supplied does not match a digest calculated " "from the uploaded file." ) # TODO: Check the file to make sure it is a valid distribution file. # Check that if it's a binary wheel, it's on a supported platform if filename.endswith(".whl"): wheel_info = _wheel_file_re.match(filename) plats = wheel_info.group("plat").split(".") if set(plats) - ALLOWED_PLATFORMS: raise _exc_with_message(HTTPBadRequest, "Binary wheel for an unsupported platform.") # Also buffer the entire signature file to disk. if "gpg_signature" in request.POST: has_signature = True with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp: signature_size = 0 for chunk in iter(lambda: request.POST["gpg_signature"].file.read(8096), b""): signature_size += len(chunk) if signature_size > MAX_SIGSIZE: raise _exc_with_message(HTTPBadRequest, "Signature too large.") fp.write(chunk) # Check whether signature is ASCII armored with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp: if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"): raise _exc_with_message(HTTPBadRequest, "PGP signature is not ASCII armored.") else: has_signature = False # TODO: We need some sort of trigger that will automatically add # filenames to Filename instead of relying on this code running # inside of our upload API. request.db.add(Filename(filename=filename)) # Store the information about the file in the database. file_ = File( release=release, filename=filename, python_version=form.pyversion.data, packagetype=form.filetype.data, comment_text=form.comment.data, size=file_size, has_signature=bool(has_signature), md5_digest=form.md5_digest.data, ) request.db.add(file_) # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is # commited. storage = request.find_service(IFileStorage) storage.store(file_.path, os.path.join(tmpdir, filename)) if has_signature: storage.store(file_.pgp_path, os.path.join(tmpdir, filename + ".asc")) return Response()
def file_upload(request): # If we're in read-only mode, let upload clients know if request.flags.enabled("read-only"): raise _exc_with_message( HTTPForbidden, "Read-only mode: Uploads are temporarily disabled") # Log an attempt to upload metrics = request.find_service(IMetricsService, context=None) metrics.increment("warehouse.upload.attempt") # Before we do anything, if there isn't an authenticated user with this # request, then we'll go ahead and bomb out. if request.authenticated_userid is None: raise _exc_with_message( HTTPForbidden, "Invalid or non-existent authentication information.") # Ensure that user has a verified, primary email address. This should both # reduce the ease of spam account creation and activity, as well as act as # a forcing function for https://github.com/pypa/warehouse/issues/3632. # TODO: Once https://github.com/pypa/warehouse/issues/3632 has been solved, # we might consider a different condition, possibly looking at # User.is_active instead. if not (request.user.primary_email and request.user.primary_email.verified): raise _exc_with_message( HTTPBadRequest, ("User {!r} does not have a verified primary email address. " "Please add a verified primary email before attempting to " "upload to PyPI. See {project_help} for more information." "for more information.").format( request.user.username, project_help=request.help_url(_anchor="verified-email"), ), ) from None # Do some cleanup of the various form fields for key in list(request.POST): value = request.POST.get(key) if isinstance(value, str): # distutils "helpfully" substitutes unknown, but "required" values # with the string "UNKNOWN". This is basically never what anyone # actually wants so we'll just go ahead and delete anything whose # value is UNKNOWN. if value.strip() == "UNKNOWN": del request.POST[key] # Escape NUL characters, which psycopg doesn't like if "\x00" in value: request.POST[key] = value.replace("\x00", "\\x00") # We require protocol_version 1, it's the only supported version however # passing a different version should raise an error. if request.POST.get("protocol_version", "1") != "1": raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.") # Check if any fields were supplied as a tuple and have become a # FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a # FieldStorage, however. # ref: https://github.com/pypa/warehouse/issues/2185 # ref: https://github.com/pypa/warehouse/issues/2491 for field in set(request.POST) - {"content", "gpg_signature"}: values = request.POST.getall(field) if any(isinstance(value, FieldStorage) for value in values): raise _exc_with_message(HTTPBadRequest, f"{field}: Should not be a tuple.") # Look up all of the valid classifiers all_classifiers = request.db.query(Classifier).all() # Validate and process the incoming metadata. form = MetadataForm(request.POST) # Add a validator for deprecated classifiers form.classifiers.validators.append(_no_deprecated_classifiers(request)) form.classifiers.choices = [(c.classifier, c.classifier) for c in all_classifiers] if not form.validate(): for field_name in _error_message_order: if field_name in form.errors: break else: field_name = sorted(form.errors.keys())[0] if field_name in form: field = form[field_name] if field.description and isinstance(field, wtforms.StringField): error_message = ( "{value!r} is an invalid value for {field}. ".format( value=field.data, field=field.description) + "Error: {} ".format(form.errors[field_name][0]) + "See " "https://packaging.python.org/specifications/core-metadata" ) else: error_message = "Invalid value for {field}. Error: {msgs[0]}".format( field=field_name, msgs=form.errors[field_name]) else: error_message = "Error: {}".format(form.errors[field_name][0]) raise _exc_with_message(HTTPBadRequest, error_message) # Ensure that we have file data in the request. if "content" not in request.POST: raise _exc_with_message(HTTPBadRequest, "Upload payload does not have a file.") # Look up the project first before doing anything else, this is so we can # automatically register it if we need to and can check permissions before # going any further. try: project = (request.db.query(Project).filter( Project.normalized_name == func.normalize_pep426_name( form.name.data)).one()) except NoResultFound: # Check for AdminFlag set by a PyPI Administrator disabling new project # registration, reasons for this include Spammers, security # vulnerabilities, or just wanting to be lazy and not worry ;) if request.flags.enabled("disallow-new-project-registration"): raise _exc_with_message( HTTPForbidden, ("New project registration temporarily disabled. " "See {projecthelp} for details").format( projecthelp=request.help_url( _anchor="admin-intervention")), ) from None # Before we create the project, we're going to check our blacklist to # see if this project is even allowed to be registered. If it is not, # then we're going to deny the request to create this project. if request.db.query(exists().where( BlacklistedProject.name == func.normalize_pep426_name( form.name.data))).scalar(): raise _exc_with_message( HTTPBadRequest, ("The name {name!r} isn't allowed. " "See {projecthelp} " "for more information.").format( name=form.name.data, projecthelp=request.help_url(_anchor="project-name"), ), ) from None # Also check for collisions with Python Standard Library modules. if packaging.utils.canonicalize_name( form.name.data) in STDLIB_PROHIBITTED: raise _exc_with_message( HTTPBadRequest, ("The name {name!r} isn't allowed (conflict with Python " "Standard Library module name). See " "{projecthelp} for more information.").format( name=form.name.data, projecthelp=request.help_url(_anchor="project-name"), ), ) from None # The project doesn't exist in our database, so first we'll check for # projects with a similar name squattees = (request.db.query(Project).filter( func.levenshtein(Project.normalized_name, func.normalize_pep426_name(form.name.data)) <= 2). all()) # Next we'll create the project project = Project(name=form.name.data) request.db.add(project) # Now that the project exists, add any squats which it is the squatter for for squattee in squattees: request.db.add(Squat(squatter=project, squattee=squattee)) # Then we'll add a role setting the current user as the "Owner" of the # project. request.db.add( Role(user=request.user, project=project, role_name="Owner")) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=project.name, action="create", submitted_by=request.user, submitted_from=request.remote_addr, )) request.db.add( JournalEntry( name=project.name, action="add Owner {}".format(request.user.username), submitted_by=request.user, submitted_from=request.remote_addr, )) # Check that the user has permission to do things to this project, if this # is a new project this will act as a sanity check for the role we just # added above. if not request.has_permission("upload", project): raise _exc_with_message( HTTPForbidden, ("The credential associated with user '{0}' " "isn't allowed to upload to project '{1}'. " "See {2} for more information.").format( request.user.username, project.name, request.help_url(_anchor="project-name"), ), ) # Update name if it differs but is still equivalent. We don't need to check if # they are equivalent when normalized because that's already been done when we # queried for the project. if project.name != form.name.data: project.name = form.name.data # Render our description so we can save from having to render this data every time # we load a project description page. rendered = None if form.description.data: description_content_type = form.description_content_type.data if not description_content_type: description_content_type = "text/x-rst" rendered = readme.render(form.description.data, description_content_type, use_fallback=False) # Uploading should prevent broken rendered descriptions. if rendered is None: if form.description_content_type.data: message = ( "The description failed to render " "for '{description_content_type}'.").format( description_content_type=description_content_type) else: message = ("The description failed to render " "in the default format of reStructuredText.") raise _exc_with_message( HTTPBadRequest, "{message} See {projecthelp} for more information.".format( message=message, projecthelp=request.help_url( _anchor="description-content-type"), ), ) from None try: canonical_version = packaging.utils.canonicalize_version( form.version.data) release = (request.db.query(Release).filter( (Release.project == project) & (Release.canonical_version == canonical_version)).one()) except MultipleResultsFound: # There are multiple releases of this project which have the same # canonical version that were uploaded before we checked for # canonical version equivalence, so return the exact match instead release = (request.db.query( Release).filter((Release.project == project) & (Release.version == form.version.data)).one()) except NoResultFound: release = Release( project=project, _classifiers=[ c for c in all_classifiers if c.classifier in form.classifiers.data ], dependencies=list( _construct_dependencies( form, { "requires": DependencyKind.requires, "provides": DependencyKind.provides, "obsoletes": DependencyKind.obsoletes, "requires_dist": DependencyKind.requires_dist, "provides_dist": DependencyKind.provides_dist, "obsoletes_dist": DependencyKind.obsoletes_dist, "requires_external": DependencyKind.requires_external, "project_urls": DependencyKind.project_url, }, )), canonical_version=canonical_version, description=Description( content_type=form.description_content_type.data, raw=form.description.data or "", html=rendered or "", rendered_by=readme.renderer_version(), ), **{ k: getattr(form, k).data for k in { # This is a list of all the fields in the form that we # should pull off and insert into our new release. "version", "summary", "license", "author", "author_email", "maintainer", "maintainer_email", "keywords", "platform", "home_page", "download_url", "requires_python", } }, uploader=request.user, uploaded_via=request.user_agent, ) request.db.add(release) # TODO: This should be handled by some sort of database trigger or # a SQLAlchemy hook or the like instead of doing it inline in # this view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="new release", submitted_by=request.user, submitted_from=request.remote_addr, )) # TODO: We need a better solution to this than to just do it inline inside # this method. Ideally the version field would just be sortable, but # at least this should be some sort of hook or trigger. releases = (request.db.query(Release).filter( Release.project == project).options( orm.load_only(Release._pypi_ordering)).all()) for i, r in enumerate( sorted(releases, key=lambda x: packaging.version.parse(x.version))): r._pypi_ordering = i # Pull the filename out of our POST data. filename = request.POST["content"].filename # Make sure that the filename does not contain any path separators. if "/" in filename or "\\" in filename: raise _exc_with_message( HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name.") # Make sure the filename ends with an allowed extension. if _dist_file_regexes[project.allow_legacy_files].search(filename) is None: raise _exc_with_message( HTTPBadRequest, "Invalid file extension: Use .egg, .tar.gz, .whl or .zip " "extension. (https://www.python.org/dev/peps/pep-0527)", ) # Make sure that our filename matches the project that it is being uploaded # to. prefix = pkg_resources.safe_name(project.name).lower() if not pkg_resources.safe_name(filename).lower().startswith(prefix): raise _exc_with_message( HTTPBadRequest, "Start filename for {!r} with {!r}.".format(project.name, prefix), ) # Check the content type of what is being uploaded if not request.POST["content"].type or request.POST[ "content"].type.startswith("image/"): raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.") # Ensure that the package filetype is allowed. # TODO: Once PEP 527 is completely implemented we should be able to delete # this and just move it into the form itself. if not project.allow_legacy_files and form.filetype.data not in { "sdist", "bdist_wheel", "bdist_egg", }: raise _exc_with_message(HTTPBadRequest, "Unknown type of file.") # The project may or may not have a file size specified on the project, if # it does then it may or may not be smaller or larger than our global file # size limits. file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit])) with tempfile.TemporaryDirectory() as tmpdir: temporary_filename = os.path.join(tmpdir, filename) # Buffer the entire file onto disk, checking the hash of the file as we # go along. with open(temporary_filename, "wb") as fp: file_size = 0 file_hashes = { "md5": hashlib.md5(), "sha256": hashlib.sha256(), "blake2_256": hashlib.blake2b(digest_size=256 // 8), } for chunk in iter(lambda: request.POST["content"].file.read(8096), b""): file_size += len(chunk) if file_size > file_size_limit: raise _exc_with_message( HTTPBadRequest, "File too large. " + "Limit for project {name!r} is {limit} MB. ".format( name=project.name, limit=file_size_limit // (1024 * 1024)) + "See " + request.help_url(_anchor="file-size-limit"), ) fp.write(chunk) for hasher in file_hashes.values(): hasher.update(chunk) # Take our hash functions and compute the final hashes for them now. file_hashes = { k: h.hexdigest().lower() for k, h in file_hashes.items() } # Actually verify the digests that we've gotten. We're going to use # hmac.compare_digest even though we probably don't actually need to # because it's better safe than sorry. In the case of multiple digests # we expect them all to be given. if not all([ hmac.compare_digest( getattr(form, "{}_digest".format(digest_name)).data.lower(), digest_value, ) for digest_name, digest_value in file_hashes.items() if getattr(form, "{}_digest".format(digest_name)).data ]): raise _exc_with_message( HTTPBadRequest, "The digest supplied does not match a digest calculated " "from the uploaded file.", ) # Check to see if the file that was uploaded exists already or not. is_duplicate = _is_duplicate_file(request.db, filename, file_hashes) if is_duplicate: return Response() elif is_duplicate is not None: raise _exc_with_message( HTTPBadRequest, # Note: Changing this error message to something that doesn't # start with "File already exists" will break the # --skip-existing functionality in twine # ref: https://github.com/pypa/warehouse/issues/3482 # ref: https://github.com/pypa/twine/issues/332 "File already exists. See " + request.help_url(_anchor="file-name-reuse"), ) # Check to see if the file that was uploaded exists in our filename log if request.db.query( request.db.query(Filename).filter( Filename.filename == filename).exists()).scalar(): raise _exc_with_message( HTTPBadRequest, "This filename has already been used, use a " "different version. " "See " + request.help_url(_anchor="file-name-reuse"), ) # Check to see if uploading this file would create a duplicate sdist # for the current release. if (form.filetype.data == "sdist" and request.db.query( request.db.query(File).filter((File.release == release) & ( File.packagetype == "sdist")).exists()).scalar()): raise _exc_with_message( HTTPBadRequest, "Only one sdist may be uploaded per release.") # Check the file to make sure it is a valid distribution file. if not _is_valid_dist_file(temporary_filename, form.filetype.data): raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.") # Check that if it's a binary wheel, it's on a supported platform if filename.endswith(".whl"): wheel_info = _wheel_file_re.match(filename) plats = wheel_info.group("plat").split(".") for plat in plats: if not _valid_platform_tag(plat): raise _exc_with_message( HTTPBadRequest, "Binary wheel '{filename}' has an unsupported " "platform tag '{plat}'.".format(filename=filename, plat=plat), ) # Also buffer the entire signature file to disk. if "gpg_signature" in request.POST: has_signature = True with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp: signature_size = 0 for chunk in iter( lambda: request.POST["gpg_signature"].file.read(8096), b""): signature_size += len(chunk) if signature_size > MAX_SIGSIZE: raise _exc_with_message(HTTPBadRequest, "Signature too large.") fp.write(chunk) # Check whether signature is ASCII armored with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp: if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"): raise _exc_with_message( HTTPBadRequest, "PGP signature isn't ASCII armored.") else: has_signature = False # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add(Filename(filename=filename)) # Store the information about the file in the database. file_ = File( release=release, filename=filename, python_version=form.pyversion.data, packagetype=form.filetype.data, comment_text=form.comment.data, size=file_size, has_signature=bool(has_signature), md5_digest=file_hashes["md5"], sha256_digest=file_hashes["sha256"], blake2_256_digest=file_hashes["blake2_256"], # Figure out what our filepath is going to be, we're going to use a # directory structure based on the hash of the file contents. This # will ensure that the contents of the file cannot change without # it also changing the path that the file is saved too. path="/".join([ file_hashes[PATH_HASHER][:2], file_hashes[PATH_HASHER][2:4], file_hashes[PATH_HASHER][4:], filename, ]), uploaded_via=request.user_agent, ) request.db.add(file_) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="add {python_version} file {filename}".format( python_version=file_.python_version, filename=file_.filename), submitted_by=request.user, submitted_from=request.remote_addr, )) # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is # committed. storage = request.find_service(IFileStorage) storage.store( file_.path, os.path.join(tmpdir, filename), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) if has_signature: storage.store( file_.pgp_path, os.path.join(tmpdir, filename + ".asc"), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) # Log a successful upload metrics.increment("warehouse.upload.ok", tags=[f"filetype:{form.filetype.data}"]) return Response()
def file_upload(request): # If we're in read-only mode, let upload clients know if request.flags.enabled("read-only"): raise _exc_with_message( HTTPForbidden, "Read-only mode: Uploads are temporarily disabled" ) # Before we do anything, if there isn't an authenticated user with this # request, then we'll go ahead and bomb out. if request.authenticated_userid is None: raise _exc_with_message( HTTPForbidden, "Invalid or non-existent authentication information." ) # Ensure that user has a verified, primary email address. This should both # reduce the ease of spam account creation and activty, as well as act as # a forcing function for https://github.com/pypa/warehouse/issues/3632. # TODO: Once https://github.com/pypa/warehouse/issues/3632 has been solved, # we might consider a different condition, possibly looking at # User.is_active instead. if not (request.user.primary_email and request.user.primary_email.verified): raise _exc_with_message( HTTPBadRequest, ( "User {!r} does not have a verified primary email address. " "Please add a verified primary email before attempting to " "upload to PyPI. See {project_help} for more information." "for more information." ).format( request.user.username, project_help=request.help_url(_anchor="verified-email"), ), ) from None # Do some cleanup of the various form fields for key in list(request.POST): value = request.POST.get(key) if isinstance(value, str): # distutils "helpfully" substitutes unknown, but "required" values # with the string "UNKNOWN". This is basically never what anyone # actually wants so we'll just go ahead and delete anything whose # value is UNKNOWN. if value.strip() == "UNKNOWN": del request.POST[key] # Escape NUL characters, which psycopg doesn't like if "\x00" in value: request.POST[key] = value.replace("\x00", "\\x00") # We require protocol_version 1, it's the only supported version however # passing a different version should raise an error. if request.POST.get("protocol_version", "1") != "1": raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.") # Check if any fields were supplied as a tuple and have become a # FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a # FieldStorage, however. # ref: https://github.com/pypa/warehouse/issues/2185 # ref: https://github.com/pypa/warehouse/issues/2491 for field in set(request.POST) - {"content", "gpg_signature"}: values = request.POST.getall(field) if any(isinstance(value, FieldStorage) for value in values): raise _exc_with_message(HTTPBadRequest, f"{field}: Should not be a tuple.") # Look up all of the valid classifiers all_classifiers = request.db.query(Classifier).all() # Validate and process the incoming metadata. form = MetadataForm(request.POST) # Add a validator for deprecated classifiers form.classifiers.validators.append(_no_deprecated_classifiers(request)) form.classifiers.choices = [(c.classifier, c.classifier) for c in all_classifiers] if not form.validate(): for field_name in _error_message_order: if field_name in form.errors: break else: field_name = sorted(form.errors.keys())[0] if field_name in form: field = form[field_name] if field.description and isinstance(field, wtforms.StringField): error_message = ( "{value!r} is an invalid value for {field}. ".format( value=field.data, field=field.description ) + "Error: {} ".format(form.errors[field_name][0]) + "See " "https://packaging.python.org/specifications/core-metadata" ) else: error_message = "Invalid value for {field}. Error: {msgs[0]}".format( field=field_name, msgs=form.errors[field_name] ) else: error_message = "Error: {}".format(form.errors[field_name][0]) raise _exc_with_message(HTTPBadRequest, error_message) # Ensure that we have file data in the request. if "content" not in request.POST: raise _exc_with_message(HTTPBadRequest, "Upload payload does not have a file.") # Look up the project first before doing anything else, this is so we can # automatically register it if we need to and can check permissions before # going any further. try: project = ( request.db.query(Project) .filter( Project.normalized_name == func.normalize_pep426_name(form.name.data) ) .one() ) except NoResultFound: # Check for AdminFlag set by a PyPI Administrator disabling new project # registration, reasons for this include Spammers, security # vulnerabilities, or just wanting to be lazy and not worry ;) if request.flags.enabled("disallow-new-project-registration"): raise _exc_with_message( HTTPForbidden, ( "New project registration temporarily disabled. " "See {projecthelp} for details" ).format(projecthelp=request.help_url(_anchor="admin-intervention")), ) from None # Before we create the project, we're going to check our blacklist to # see if this project is even allowed to be registered. If it is not, # then we're going to deny the request to create this project. if request.db.query( exists().where( BlacklistedProject.name == func.normalize_pep426_name(form.name.data) ) ).scalar(): raise _exc_with_message( HTTPBadRequest, ( "The name {name!r} isn't allowed. " "See {projecthelp} " "for more information." ).format( name=form.name.data, projecthelp=request.help_url(_anchor="project-name"), ), ) from None # Also check for collisions with Python Standard Library modules. if packaging.utils.canonicalize_name(form.name.data) in STDLIB_PROHIBITTED: raise _exc_with_message( HTTPBadRequest, ( "The name {name!r} isn't allowed (conflict with Python " "Standard Library module name). See " "{projecthelp} for more information." ).format( name=form.name.data, projecthelp=request.help_url(_anchor="project-name"), ), ) from None # The project doesn't exist in our database, so first we'll check for # projects with a similar name squattees = ( request.db.query(Project) .filter( func.levenshtein( Project.normalized_name, func.normalize_pep426_name(form.name.data) ) <= 2 ) .all() ) # Next we'll create the project project = Project(name=form.name.data) request.db.add(project) # Now that the project exists, add any squats which it is the squatter for for squattee in squattees: request.db.add(Squat(squatter=project, squattee=squattee)) # Then we'll add a role setting the current user as the "Owner" of the # project. request.db.add(Role(user=request.user, project=project, role_name="Owner")) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=project.name, action="create", submitted_by=request.user, submitted_from=request.remote_addr, ) ) request.db.add( JournalEntry( name=project.name, action="add Owner {}".format(request.user.username), submitted_by=request.user, submitted_from=request.remote_addr, ) ) # Check that the user has permission to do things to this project, if this # is a new project this will act as a sanity check for the role we just # added above. if not request.has_permission("upload", project): raise _exc_with_message( HTTPForbidden, ( "The user '{0}' isn't allowed to upload to project '{1}'. " "See {2} for more information." ).format( request.user.username, project.name, request.help_url(_anchor="project-name"), ), ) # Uploading should prevent broken rendered descriptions. # Temporarily disabled, see # https://github.com/pypa/warehouse/issues/4079 # if form.description.data: # description_content_type = form.description_content_type.data # if not description_content_type: # description_content_type = "text/x-rst" # rendered = readme.render( # form.description.data, description_content_type, use_fallback=False # ) # if rendered is None: # if form.description_content_type.data: # message = ( # "The description failed to render " # "for '{description_content_type}'." # ).format(description_content_type=description_content_type) # else: # message = ( # "The description failed to render " # "in the default format of reStructuredText." # ) # raise _exc_with_message( # HTTPBadRequest, # "{message} See {projecthelp} for more information.".format( # message=message, # projecthelp=request.help_url(_anchor="description-content-type"), # ), # ) from None try: canonical_version = packaging.utils.canonicalize_version(form.version.data) release = ( request.db.query(Release) .filter( (Release.project == project) & (Release.canonical_version == canonical_version) ) .one() ) except MultipleResultsFound: # There are multiple releases of this project which have the same # canonical version that were uploaded before we checked for # canonical version equivalence, so return the exact match instead release = ( request.db.query(Release) .filter( (Release.project == project) & (Release.version == form.version.data) ) .one() ) except NoResultFound: release = Release( project=project, _classifiers=[ c for c in all_classifiers if c.classifier in form.classifiers.data ], dependencies=list( _construct_dependencies( form, { "requires": DependencyKind.requires, "provides": DependencyKind.provides, "obsoletes": DependencyKind.obsoletes, "requires_dist": DependencyKind.requires_dist, "provides_dist": DependencyKind.provides_dist, "obsoletes_dist": DependencyKind.obsoletes_dist, "requires_external": DependencyKind.requires_external, "project_urls": DependencyKind.project_url, }, ) ), canonical_version=canonical_version, **{ k: getattr(form, k).data for k in { # This is a list of all the fields in the form that we # should pull off and insert into our new release. "version", "summary", "description", "description_content_type", "license", "author", "author_email", "maintainer", "maintainer_email", "keywords", "platform", "home_page", "download_url", "requires_python", } }, uploader=request.user, uploaded_via=request.user_agent, ) request.db.add(release) # TODO: This should be handled by some sort of database trigger or # a SQLAlchemy hook or the like instead of doing it inline in # this view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="new release", submitted_by=request.user, submitted_from=request.remote_addr, ) ) # TODO: We need a better solution to this than to just do it inline inside # this method. Ideally the version field would just be sortable, but # at least this should be some sort of hook or trigger. releases = ( request.db.query(Release) .filter(Release.project == project) .options(orm.load_only(Release._pypi_ordering)) .all() ) for i, r in enumerate( sorted(releases, key=lambda x: packaging.version.parse(x.version)) ): r._pypi_ordering = i # Pull the filename out of our POST data. filename = request.POST["content"].filename # Make sure that the filename does not contain any path separators. if "/" in filename or "\\" in filename: raise _exc_with_message( HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name." ) # Make sure the filename ends with an allowed extension. if _dist_file_regexes[project.allow_legacy_files].search(filename) is None: raise _exc_with_message( HTTPBadRequest, "Invalid file extension: Use .egg, .tar.gz, .whl or .zip " "extension. (https://www.python.org/dev/peps/pep-0527)", ) # Make sure that our filename matches the project that it is being uploaded # to. prefix = pkg_resources.safe_name(project.name).lower() if not pkg_resources.safe_name(filename).lower().startswith(prefix): raise _exc_with_message( HTTPBadRequest, "Start filename for {!r} with {!r}.".format(project.name, prefix), ) # Check the content type of what is being uploaded if not request.POST["content"].type or request.POST["content"].type.startswith( "image/" ): raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.") # Ensure that the package filetype is allowed. # TODO: Once PEP 527 is completely implemented we should be able to delete # this and just move it into the form itself. if not project.allow_legacy_files and form.filetype.data not in { "sdist", "bdist_wheel", "bdist_egg", }: raise _exc_with_message(HTTPBadRequest, "Unknown type of file.") # The project may or may not have a file size specified on the project, if # it does then it may or may not be smaller or larger than our global file # size limits. file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit])) with tempfile.TemporaryDirectory() as tmpdir: temporary_filename = os.path.join(tmpdir, filename) # Buffer the entire file onto disk, checking the hash of the file as we # go along. with open(temporary_filename, "wb") as fp: file_size = 0 file_hashes = { "md5": hashlib.md5(), "sha256": hashlib.sha256(), "blake2_256": hashlib.blake2b(digest_size=256 // 8), } for chunk in iter(lambda: request.POST["content"].file.read(8096), b""): file_size += len(chunk) if file_size > file_size_limit: raise _exc_with_message( HTTPBadRequest, "File too large. " + "Limit for project {name!r} is {limit} MB. ".format( name=project.name, limit=file_size_limit // (1024 * 1024) ) + "See " + request.help_url(_anchor="file-size-limit"), ) fp.write(chunk) for hasher in file_hashes.values(): hasher.update(chunk) # Take our hash functions and compute the final hashes for them now. file_hashes = {k: h.hexdigest().lower() for k, h in file_hashes.items()} # Actually verify the digests that we've gotten. We're going to use # hmac.compare_digest even though we probably don't actually need to # because it's better safe than sorry. In the case of multiple digests # we expect them all to be given. if not all( [ hmac.compare_digest( getattr(form, "{}_digest".format(digest_name)).data.lower(), digest_value, ) for digest_name, digest_value in file_hashes.items() if getattr(form, "{}_digest".format(digest_name)).data ] ): raise _exc_with_message( HTTPBadRequest, "The digest supplied does not match a digest calculated " "from the uploaded file.", ) # Check to see if the file that was uploaded exists already or not. is_duplicate = _is_duplicate_file(request.db, filename, file_hashes) if is_duplicate: return Response() elif is_duplicate is not None: raise _exc_with_message( HTTPBadRequest, # Note: Changing this error message to something that doesn't # start with "File already exists" will break the # --skip-existing functionality in twine # ref: https://github.com/pypa/warehouse/issues/3482 # ref: https://github.com/pypa/twine/issues/332 "File already exists. See " + request.help_url(_anchor="file-name-reuse"), ) # Check to see if the file that was uploaded exists in our filename log if request.db.query( request.db.query(Filename).filter(Filename.filename == filename).exists() ).scalar(): raise _exc_with_message( HTTPBadRequest, "This filename has already been used, use a " "different version. " "See " + request.help_url(_anchor="file-name-reuse"), ) # Check to see if uploading this file would create a duplicate sdist # for the current release. if ( form.filetype.data == "sdist" and request.db.query( request.db.query(File) .filter((File.release == release) & (File.packagetype == "sdist")) .exists() ).scalar() ): raise _exc_with_message( HTTPBadRequest, "Only one sdist may be uploaded per release." ) # Check the file to make sure it is a valid distribution file. if not _is_valid_dist_file(temporary_filename, form.filetype.data): raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.") # Check that if it's a binary wheel, it's on a supported platform if filename.endswith(".whl"): wheel_info = _wheel_file_re.match(filename) plats = wheel_info.group("plat").split(".") for plat in plats: if not _valid_platform_tag(plat): raise _exc_with_message( HTTPBadRequest, "Binary wheel '{filename}' has an unsupported " "platform tag '{plat}'.".format(filename=filename, plat=plat), ) # Also buffer the entire signature file to disk. if "gpg_signature" in request.POST: has_signature = True with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp: signature_size = 0 for chunk in iter( lambda: request.POST["gpg_signature"].file.read(8096), b"" ): signature_size += len(chunk) if signature_size > MAX_SIGSIZE: raise _exc_with_message(HTTPBadRequest, "Signature too large.") fp.write(chunk) # Check whether signature is ASCII armored with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp: if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"): raise _exc_with_message( HTTPBadRequest, "PGP signature isn't ASCII armored." ) else: has_signature = False # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add(Filename(filename=filename)) # Store the information about the file in the database. file_ = File( release=release, filename=filename, python_version=form.pyversion.data, packagetype=form.filetype.data, comment_text=form.comment.data, size=file_size, has_signature=bool(has_signature), md5_digest=file_hashes["md5"], sha256_digest=file_hashes["sha256"], blake2_256_digest=file_hashes["blake2_256"], # Figure out what our filepath is going to be, we're going to use a # directory structure based on the hash of the file contents. This # will ensure that the contents of the file cannot change without # it also changing the path that the file is saved too. path="/".join( [ file_hashes[PATH_HASHER][:2], file_hashes[PATH_HASHER][2:4], file_hashes[PATH_HASHER][4:], filename, ] ), uploaded_via=request.user_agent, ) request.db.add(file_) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. request.db.add( JournalEntry( name=release.project.name, version=release.version, action="add {python_version} file {filename}".format( python_version=file_.python_version, filename=file_.filename ), submitted_by=request.user, submitted_from=request.remote_addr, ) ) # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is # committed. storage = request.find_service(IFileStorage) storage.store( file_.path, os.path.join(tmpdir, filename), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) if has_signature: storage.store( file_.pgp_path, os.path.join(tmpdir, filename + ".asc"), meta={ "project": file_.release.project.normalized_name, "version": file_.release.version, "package-type": file_.packagetype, "python-version": file_.python_version, }, ) return Response()