def release_prohibited_project_name(request):
project_name = request.POST.get("project_name")
if project_name is None:
request.session.flash("Provide a project name", queue="error")
return HTTPSeeOther(request.current_route_path())
prohibited_project_name = (
request.db.query(ProhibitedProjectName)
.filter(ProhibitedProjectName.name == func.normalize_pep426_name(project_name))
.first()
)
if prohibited_project_name is None:
request.session.flash(
f"{project_name!r} does not exist on prohibited project name list.",
queue="error",
)
return HTTPSeeOther(request.current_route_path())
project = (
request.db.query(Project)
.filter(Project.normalized_name == func.normalize_pep426_name(project_name))
.first()
)
if project is not None:
request.session.flash(
f"{project_name!r} exists and is not on the prohibited project name list.",
queue="error",
)
return HTTPSeeOther(request.current_route_path())
username = request.POST.get("username")
if not username:
request.session.flash("Provide a username", queue="error")
return HTTPSeeOther(request.current_route_path())
user = request.db.query(User).filter(User.username == username).first()
if user is None:
request.session.flash(f"Unknown username '{username}'", queue="error")
return HTTPSeeOther(request.current_route_path())
project = Project(name=project_name)
request.db.add(project)
request.db.add(Role(project=project, user=user, role_name="Owner"))
request.db.delete(prohibited_project_name)
request.session.flash(
f"{project.name!r} released to {user.username!r}.", queue="success"
)
request.db.flush()
return HTTPSeeOther(
request.route_path("admin.project.detail", project_name=project.normalized_name)
)
def release_urls(request, package_name: str, version: str):
files = (
request.db.query(File)
.join(Release, Project)
.filter(
(Project.normalized_name == func.normalize_pep426_name(package_name))
& (Release.version == version)
)
.all()
)
return [
{
"filename": f.filename,
"packagetype": f.packagetype,
"python_version": f.python_version,
"size": f.size,
"md5_digest": f.md5_digest,
"sha256_digest": f.sha256_digest,
"digests": {"md5": f.md5_digest, "sha256": f.sha256_digest},
"has_sig": f.has_signature,
"upload_time": f.upload_time.isoformat() + "Z",
"comment_text": f.comment_text,
# TODO: Remove this once we've had a long enough time with it
# here to consider it no longer in use.
"downloads": -1,
"path": f.path,
"url": request.route_url("packaging.file", path=f.path),
}
for f in files
]
def __getitem__(self, project):
try:
return self.request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(project)
).one()
except NoResultFound:
raise KeyError from None
def release_urls(request, package_name, version):
files = (request.db.query(File).join(Release, Project).filter(
(Project.normalized_name == func.normalize_pep426_name(package_name))
& (Release.version == version)).all())
return [
{
"filename": f.filename,
"packagetype": f.packagetype,
"python_version": f.python_version,
"size": f.size,
"md5_digest": f.md5_digest,
"sha256_digest": f.sha256_digest,
"digests": {
"md5": f.md5_digest,
"sha256": f.sha256_digest,
},
"has_sig": f.has_signature,
"upload_time": f.upload_time.isoformat() + "Z",
"comment_text": f.comment_text,
# TODO: Remove this once we've had a long enough time with it
# here to consider it no longer in use.
"downloads": -1,
"path": f.path,
"url": request.route_url("packaging.file", path=f.path),
} for f in files
]
def package_roles(request, package_name):
roles = (request.db.query(Role).join(
User,
Project).filter(Project.normalized_name == func.normalize_pep426_name(
package_name)).order_by(Role.role_name.desc(),
User.username).all())
return [(r.role_name, r.user.username) for r in roles]
def blacklist(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
blacklist_query = request.db.query(BlacklistedProject).order_by(
BlacklistedProject.name
)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
filters.append(
BlacklistedProject.name.ilike(func.normalize_pep426_name(term))
)
blacklist_query = blacklist_query.filter(or_(*filters))
blacklist = SQLAlchemyORMPage(
blacklist_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"blacklist": blacklist, "query": q}
def add_blacklist(request):
project_name = request.POST.get("project")
if project_name is None:
raise HTTPBadRequest("Have a project to confirm.")
comment = request.POST.get("comment", "")
# Verify that the user has confirmed the request to blacklist.
confirm = request.POST.get("confirm")
if not confirm:
request.session.flash("Confirm the blacklist request", queue="error")
return HTTPSeeOther(request.current_route_path())
elif canonicalize_name(confirm) != canonicalize_name(project_name):
request.session.flash(
f"{confirm!r} is not the same as {project_name!r}", queue="error")
return HTTPSeeOther(request.current_route_path())
# Add our requested blacklist.
request.db.add(
BlacklistedProject(name=project_name,
comment=comment,
blacklisted_by=request.user))
# Go through and delete the project and everything related to it so that
# our blacklist actually blocks things and isn't ignored (since the
# blacklist only takes effect on new project registration).
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
project_name)).first())
if project is not None:
remove_project(project, request)
request.session.flash(f"Blacklisted {project_name!r}", queue="success")
return HTTPSeeOther(request.route_path("admin.blacklist.list"))
def release_urls(request, package_name, version):
files = (
request.db.query(File)
.join(Release, Project)
.filter((Project.normalized_name ==
func.normalize_pep426_name(package_name)) &
(Release.version == version))
.all()
)
return [
{
"filename": f.filename,
"packagetype": f.packagetype,
"python_version": f.python_version,
"size": f.size,
"md5_digest": f.md5_digest,
"has_sig": f.has_signature,
"upload_time": f.upload_time,
"comment_text": f.comment_text,
"downloads": f.downloads,
"url": request.route_url("packaging.file", path=f.path),
}
for f in files
]
def release_data(request, package_name, version):
try:
release = (
request.db.query(Release)
.options(orm.undefer("description"))
.join(Project)
.filter((Project.normalized_name ==
func.normalize_pep426_name(package_name)) &
(Release.version == version))
.one()
)
except NoResultFound:
return {}
stats_svc = request.find_service(IDownloadStatService)
return {
"name": release.project.name,
"version": release.version,
"stable_version": release.project.stable_version,
"bugtrack_url": release.project.bugtrack_url,
"package_url": request.route_url(
"packaging.project",
name=release.project.name,
),
"release_url": request.route_url(
"packaging.release",
name=release.project.name,
version=release.version,
),
"docs_url": release.project.documentation_url,
"home_page": release.home_page,
"download_url": release.download_url,
"project_url": list(release.project_urls),
"author": release.author,
"author_email": release.author_email,
"maintainer": release.maintainer,
"maintainer_email": release.maintainer_email,
"summary": release.summary,
"description": release.description,
"license": release.license,
"keywords": release.keywords,
"platform": release.platform,
"classifiers": list(release.classifiers),
"requires": list(release.requires),
"requires_dist": list(release.requires_dist),
"provides": list(release.provides),
"provides_dist": list(release.provides_dist),
"obsoletes": list(release.obsoletes),
"obsoletes_dist": list(release.obsoletes_dist),
"requires_python": release.requires_python,
"requires_external": list(release.requires_external),
"_pypi_ordering": release._pypi_ordering,
"_pypi_hidden": release._pypi_hidden,
"downloads": {
"last_day": stats_svc.get_daily_stats(release.project.name),
"last_week": stats_svc.get_weekly_stats(release.project.name),
"last_month": stats_svc.get_monthly_stats(release.project.name),
},
}
def __getitem__(self, project):
try:
return (self.request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
project)).one())
except NoResultFound:
raise KeyError from None
def _get_project(request, vuln_report: vulnerabilities.VulnerabilityReportRequest):
return (
request.db.query(Project)
.filter(
Project.normalized_name == func.normalize_pep426_name(vuln_report.project)
)
.one()
)
def package_roles(request, package_name: str):
roles = (
request.db.query(Role)
.join(User, Project)
.filter(Project.normalized_name == func.normalize_pep426_name(package_name))
.order_by(Role.role_name.desc(), User.username)
.all()
)
return [(r.role_name, r.user.username) for r in roles]
def package_releases(request, package_name, show_hidden=False):
# This used to support the show_hidden parameter to determine if it should
# show hidden releases or not. However, Warehouse doesn't support the
# concept of hidden releases, so it is just no-opd now and left here for
# compatibility's sake.
versions = (request.db.query(Release.version).join(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
package_name)).order_by(Release._pypi_ordering).all())
return [v[0] for v in versions]
def package_hosting_mode(request, package_name):
try:
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
package_name)).one())
except NoResultFound:
return None
else:
return project.hosting_mode
def release_data(request, package_name, version):
try:
release = (
request.db.query(Release)
.options(orm.undefer("description"))
.join(Project)
.filter(
(Project.normalized_name == func.normalize_pep426_name(package_name))
& (Release.version == version)
)
.one()
)
except NoResultFound:
return {}
return {
"name": release.project.name,
"version": release.version,
"stable_version": release.project.stable_version,
"bugtrack_url": release.project.bugtrack_url,
"package_url": request.route_url(
"packaging.project", name=release.project.name
),
"release_url": request.route_url(
"packaging.release", name=release.project.name, version=release.version
),
"docs_url": release.project.documentation_url,
"home_page": release.home_page,
"download_url": release.download_url,
"project_url": list(release.project_urls),
"author": release.author,
"author_email": release.author_email,
"maintainer": release.maintainer,
"maintainer_email": release.maintainer_email,
"summary": release.summary,
"description": release.description,
"license": release.license,
"keywords": release.keywords,
"platform": release.platform,
"classifiers": list(release.classifiers),
"requires": list(release.requires),
"requires_dist": list(release.requires_dist),
"provides": list(release.provides),
"provides_dist": list(release.provides_dist),
"obsoletes": list(release.obsoletes),
"obsoletes_dist": list(release.obsoletes_dist),
"requires_python": release.requires_python,
"requires_external": list(release.requires_external),
"_pypi_ordering": release._pypi_ordering,
"_pypi_hidden": release._pypi_hidden,
"downloads": {"last_day": -1, "last_week": -1, "last_month": -1},
"cheesecake_code_kwalitee_id": None,
"cheesecake_documentation_id": None,
"cheesecake_installability_id": None,
}
def add_prohibited_project_names(request):
project_name = request.POST.get("project")
if project_name is None:
raise HTTPBadRequest("Have a project to confirm.")
comment = request.POST.get("comment", "")
# Verify that the user has confirmed the request to prohibit.
confirm = request.POST.get("confirm")
if not confirm:
request.session.flash(
"Confirm the prohibited project name request", queue="error"
)
return HTTPSeeOther(request.current_route_path())
elif canonicalize_name(confirm) != canonicalize_name(project_name):
request.session.flash(
f"{confirm!r} is not the same as {project_name!r}", queue="error"
)
return HTTPSeeOther(request.current_route_path())
# Check to make sure the object doesn't already exist.
if (
request.db.query(literal(True))
.filter(
request.db.query(ProhibitedProjectName)
.filter(ProhibitedProjectName.name == project_name)
.exists()
)
.scalar()
):
request.session.flash(
f"{project_name!r} has already been prohibited.", queue="error"
)
return HTTPSeeOther(request.route_path("admin.prohibited_project_names.list"))
# Add our requested prohibition.
request.db.add(
ProhibitedProjectName(
name=project_name, comment=comment, prohibited_by=request.user
)
)
# Go through and delete the project and everything related to it so that
# our prohibition actually blocks things and isn't ignored (since the
# prohibition only takes effect on new project registration).
project = (
request.db.query(Project)
.filter(Project.normalized_name == func.normalize_pep426_name(project_name))
.first()
)
if project is not None:
remove_project(project, request)
request.session.flash(f"Prohibited Project Name {project_name!r}", queue="success")
return HTTPSeeOther(request.route_path("admin.prohibited_project_names.list"))
def release_data(request, package_name: str, version: str):
try:
release = (
request.db.query(Release)
.options(orm.undefer("description"))
.join(Project)
.filter(
(Project.normalized_name == func.normalize_pep426_name(package_name))
& (Release.version == version)
)
.one()
)
except NoResultFound:
return {}
return {
"name": release.project.name,
"version": release.version,
"stable_version": None,
"bugtrack_url": None,
"package_url": request.route_url(
"packaging.project", name=release.project.name
),
"release_url": request.route_url(
"packaging.release", name=release.project.name, version=release.version
),
"docs_url": _clean_for_xml(release.project.documentation_url),
"home_page": _clean_for_xml(release.home_page),
"download_url": _clean_for_xml(release.download_url),
"project_url": [_clean_for_xml(url) for url in release.project_urls],
"author": _clean_for_xml(release.author),
"author_email": _clean_for_xml(release.author_email),
"maintainer": _clean_for_xml(release.maintainer),
"maintainer_email": _clean_for_xml(release.maintainer_email),
"summary": _clean_for_xml(release.summary),
"description": _clean_for_xml(release.description),
"license": _clean_for_xml(release.license),
"keywords": _clean_for_xml(release.keywords),
"platform": release.platform,
"classifiers": list(release.classifiers),
"requires": list(release.requires),
"requires_dist": list(release.requires_dist),
"provides": list(release.provides),
"provides_dist": list(release.provides_dist),
"obsoletes": list(release.obsoletes),
"obsoletes_dist": list(release.obsoletes_dist),
"requires_python": release.requires_python,
"requires_external": list(release.requires_external),
"_pypi_ordering": release._pypi_ordering,
"downloads": {"last_day": -1, "last_week": -1, "last_month": -1},
"cheesecake_code_kwalitee_id": None,
"cheesecake_documentation_id": None,
"cheesecake_installability_id": None,
}
def __getitem__(self, organization):
# Try returning organization with matching name.
try:
return (self.request.db.query(Organization).filter(
Organization.normalized_name == func.normalize_pep426_name(
organization)).one())
except NoResultFound:
pass
# Try redirecting to a renamed organization.
try:
organization = (self.request.db.query(Organization).join(
OrganizationNameCatalog,
OrganizationNameCatalog.organization_id == Organization.id,
).filter(OrganizationNameCatalog.normalized_name ==
func.normalize_pep426_name(organization)).one())
raise HTTPPermanentRedirect(
self.request.matched_route.generate(
{"organization_name": organization.normalized_name}))
except NoResultFound:
raise KeyError from None
def package_hosting_mode(request, package_name):
try:
project = (
request.db.query(Project)
.filter(Project.normalized_name ==
func.normalize_pep426_name(package_name))
.one()
)
except NoResultFound:
return None
else:
return project.hosting_mode
def package_releases(request, package_name, show_hidden=False):
# This used to support the show_hidden parameter to determine if it should
# show hidden releases or not. However, Warehouse doesn't support the
# concept of hidden releases, so this parameter controls if the latest
# version or all_versions are returned.
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
package_name)).one())
if show_hidden:
return [v.version for v in project.all_versions]
else:
return [project.latest_version.version]
def add_blacklist(request):
project_name = request.POST.get("project")
if project_name is None:
raise HTTPBadRequest("Have a project to confirm.")
comment = request.POST.get("comment", "")
# Verify that the user has confirmed the request to blacklist.
confirm = request.POST.get("confirm")
if not confirm:
request.session.flash("Confirm the blacklist request", queue="error")
return HTTPSeeOther(request.current_route_path())
elif canonicalize_name(confirm) != canonicalize_name(project_name):
request.session.flash(
f"{confirm!r} is not the same as {project_name!r}", queue="error"
)
return HTTPSeeOther(request.current_route_path())
# Check to make sure the object doesn't already exist.
if (
request.db.query(literal(True))
.filter(
request.db.query(BlacklistedProject)
.filter(BlacklistedProject.name == project_name)
.exists()
)
.scalar()
):
request.session.flash(
f"{project_name!r} has already been blacklisted.", queue="error"
)
return HTTPSeeOther(request.route_path("admin.blacklist.list"))
# Add our requested blacklist.
request.db.add(
BlacklistedProject(
name=project_name, comment=comment, blacklisted_by=request.user
)
)
# Go through and delete the project and everything related to it so that
# our blacklist actually blocks things and isn't ignored (since the
# blacklist only takes effect on new project registration).
project = (
request.db.query(Project)
.filter(Project.normalized_name == func.normalize_pep426_name(project_name))
.first()
)
if project is not None:
remove_project(project, request)
request.session.flash(f"Blacklisted {project_name!r}", queue="success")
return HTTPSeeOther(request.route_path("admin.blacklist.list"))
def confirm_blacklist(request):
project_name = request.GET.get("project")
if project_name is None:
raise HTTPBadRequest("Have a project to confirm.")
comment = request.GET.get("comment", "")
# We need to look up to see if there is an existing project, releases,
# files, roles, etc for what we're attempting to blacklist. If there is we
# need to warn that blacklisting will delete those.
project = (
request.db.query(Project)
.filter(Project.normalized_name == func.normalize_pep426_name(project_name))
.first()
)
if project is not None:
releases = (
request.db.query(Release)
.join(Project)
.filter(Release.project == project)
.all()
)
files = (
request.db.query(File)
.join(Release)
.join(Project)
.filter(Release.project == project)
.all()
)
roles = (
request.db.query(Role)
.join(User)
.join(Project)
.filter(Role.project == project)
.distinct(User.username)
.order_by(User.username)
.all()
)
else:
releases = []
files = []
roles = []
return {
"blacklist": {"project": project_name, "comment": comment},
"existing": {
"project": project,
"releases": releases,
"files": files,
"roles": roles,
},
}
def package_releases(request, package_name, show_hidden=False):
# This used to support the show_hidden parameter to determine if it should
# show hidden releases or not. However, Warehouse doesn't support the
# concept of hidden releases, so it is just no-opd now and left here for
# compatibility's sake.
versions = (
request.db.query(Release.version)
.join(Project)
.filter(Project.normalized_name ==
func.normalize_pep426_name(package_name))
.order_by(Release._pypi_ordering)
.all()
)
return [v[0] for v in versions]
class Project(db.ModelBase):
__tablename__ = "packages"
__table_args__ = (CheckConstraint(
"name ~* '^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$'::text",
name="packages_valid_name",
), )
__repr__ = make_repr("name")
name = Column(Text, primary_key=True, nullable=False)
normalized_name = orm.column_property(func.normalize_pep426_name(name))
stable_version = Column(Text)
autohide = Column(Boolean, server_default=sql.true())
comments = Column(Boolean, server_default=sql.true())
bugtrack_url = Column(Text)
hosting_mode = Column(Text, nullable=False, server_default="pypi-explicit")
created = Column(
DateTime(timezone=False),
nullable=False,
server_default=sql.func.now(),
)
releases = orm.relationship(
"Release",
backref="project",
cascade="all, delete-orphan",
lazy="dynamic",
)
def __getitem__(self, version):
try:
return self.releases.filter(Release.version == version).one()
except NoResultFound:
raise KeyError from None
@property
def documentation_url(self):
# TODO: Move this into the database and elimnate the use of the
# threadlocal here.
registry = get_current_registry()
request = get_current_request()
path = "/".join([self.name, "index.html"])
# If the path doesn't exist, then we'll just return a None here.
if not registry["filesystems"]["documentation"].exists(path):
return
return request.route_url("legacy.docs", project=self.name)
def find_organizationid(self, name):
"""
Find the unique organization identifier for the given normalized name or None
if there is no organization with the given name.
"""
normalized_name = func.normalize_pep426_name(name)
try:
(organization_id, ) = (self.db.query(
OrganizationNameCatalog.organization_id).filter(
OrganizationNameCatalog.normalized_name ==
normalized_name).one())
except NoResultFound:
return
return organization_id
def bulk_add_prohibited_project_names(request):
if request.method == "POST":
project_names = request.POST.get("projects", "").split()
comment = request.POST.get("comment", "")
for project_name in project_names:
# Check to make sure the object doesn't already exist.
if (
request.db.query(literal(True))
.filter(
request.db.query(ProhibitedProjectName)
.filter(ProhibitedProjectName.name == project_name)
.exists()
)
.scalar()
):
continue
# Add our requested prohibition.
request.db.add(
ProhibitedProjectName(
name=project_name, comment=comment, prohibited_by=request.user
)
)
# Go through and delete the project and everything related to it so that
# our prohibition actually blocks things and isn't ignored (since the
# prohibition only takes effect on new project registration).
project = (
request.db.query(Project)
.filter(
Project.normalized_name == func.normalize_pep426_name(project_name)
)
.first()
)
if project is not None:
remove_project(project, request, flash=False)
request.session.flash(
f"Prohibited {len(project_names)!r} projects", queue="success"
)
return HTTPSeeOther(
request.route_path("admin.prohibited_project_names.bulk_add")
)
return {}
def release_urls(request, package_name, version):
files = (request.db.query(File).join(Release, Project).filter(
(Project.normalized_name == func.normalize_pep426_name(package_name))
& (Release.version == version)).all())
return [{
"filename": f.filename,
"packagetype": f.packagetype,
"python_version": f.python_version,
"size": f.size,
"md5_digest": f.md5_digest,
"has_sig": f.has_signature,
"upload_time": f.upload_time,
"comment_text": f.comment_text,
"downloads": f.downloads,
"url": request.route_url("packaging.file", path=f.path),
} for f in files]
def package_releases(request, package_name: str, show_hidden: bool = False):
try:
project = (
request.db.query(Project)
.filter(Project.normalized_name == func.normalize_pep426_name(package_name))
.one()
)
except NoResultFound:
return []
# This used to support the show_hidden parameter to determine if it should
# show hidden releases or not. However, Warehouse doesn't support the
# concept of hidden releases, so this parameter controls if the latest
# version or all_versions are returned.
if show_hidden:
return [v.version for v in project.all_versions]
else:
latest_version = project.latest_version
if latest_version is None:
return []
return [latest_version.version]
class Project(db.Model, Syncable):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text, nullable=False, index=True)
team_slug = db.Column(db.String(255))
normalized_name = orm.column_property(func.normalize_pep426_name(name))
description = db.Column(db.Text)
html_url = db.Column(db.String(255))
subscribers_count = db.Column(db.SmallInteger, default=0, nullable=False)
stargazers_count = db.Column(db.SmallInteger, default=0, nullable=False)
forks_count = db.Column(db.SmallInteger, default=0, nullable=False)
open_issues_count = db.Column(db.SmallInteger, default=0, nullable=False)
is_active = db.Column(db.Boolean, default=True, nullable=False, index=True)
transfer_issue_url = db.Column(db.String(255))
membership = db.relationship("ProjectMembership",
backref="project",
lazy="dynamic")
credentials = db.relationship("ProjectCredential",
backref="project",
lazy="dynamic")
uploads = db.relationship(
"ProjectUpload",
backref="project",
lazy="dynamic",
order_by=lambda: ProjectUpload.ordering.desc().nullslast(),
)
created_at = db.Column(db.DateTime, nullable=True)
updated_at = db.Column(db.DateTime, nullable=True)
pushed_at = db.Column(db.DateTime, nullable=True)
__tablename__ = "projects"
__table_args__ = (
db.Index("release_name_idx", "name"),
db.Index("release_name_is_active_idx", "name", "is_active"),
)
def __str__(self):
return self.name
@aggregated("uploads", db.Column(db.SmallInteger))
def uploads_count(self):
return db.func.count("1")
@aggregated("membership", db.Column(db.SmallInteger))
def membership_count(self):
return db.func.count("1")
@property
def current_user_is_member(self):
if not current_user:
return False
elif not current_user.is_authenticated:
return False
elif current_user_is_roadie():
return True
else:
return self.user_is_member(current_user)
@property
def current_user_is_lead(self):
if not current_user:
return False
elif not current_user.is_authenticated:
return False
elif current_user_is_roadie():
return True
else:
return current_user.id in [
user.id
for user in self.lead_members.options(orm.load_only("id"))
]
@property
def all_members(self):
return (User.active_members().join(User.projects_memberships).filter(
ProjectMembership.project_id == self.id))
@property
def nonlead_members(self):
return self.all_members.filter(ProjectMembership.is_lead.is_(False))
@property
def lead_members(self):
return self.all_members.filter(ProjectMembership.is_lead.is_(True))
def user_is_member(self, user):
return user.id in [
member.id
for member in self.all_members.options(orm.load_only("id"))
]
@property
def pypi_json_url(self):
"""
The URL to fetch JSON data from PyPI, using a timestamp to work-around
the PyPI CDN cache.
"""
return (
f"https://pypi.org/pypi/{self.normalized_name}/json?time={int(time.time())}"
)
def create_transfer_issue(self, assignees, **data):
issue_response = github.new_project_issue(
repo=self.name,
data={
"title": render_template("hooks/project-title.txt", **data),
"body": render_template("hooks/project-body.txt", **data),
"assignees": assignees,
},
)
issue_data = issue_response.json()
issue_url = issue_data.get("html_url")
if issue_url.startswith(f"https://github.com/jazzband/{self.name}"):
self.transfer_issue_url = issue_url
self.save()
def create_team(self):
team_response = github.create_project_team(self.name)
if team_response and team_response.status_code == 201:
team_data = team_response.json()
self.team_slug = team_data.get("slug")
self.save()
return team_response
class Project(db.Model, Helpers, Syncable):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text, nullable=False, index=True)
normalized_name = orm.column_property(func.normalize_pep426_name(name))
description = db.Column(db.Text)
html_url = db.Column(db.String(255))
subscribers_count = db.Column(db.SmallInteger, default=0, nullable=False)
stargazers_count = db.Column(db.SmallInteger, default=0, nullable=False)
forks_count = db.Column(db.SmallInteger, default=0, nullable=False)
open_issues_count = db.Column(db.SmallInteger, default=0, nullable=False)
uploads_count = db.Column(db.SmallInteger, default=0)
is_active = db.Column(db.Boolean, default=True, nullable=False, index=True)
membership = db.relationship('ProjectMembership',
backref='project',
lazy='dynamic')
credentials = db.relationship('ProjectCredential',
backref='project',
lazy='dynamic')
uploads = db.relationship('ProjectUpload',
backref='project',
lazy='dynamic')
created_at = db.Column(db.DateTime, nullable=True)
updated_at = db.Column(db.DateTime, nullable=True)
pushed_at = db.Column(db.DateTime, nullable=True)
__tablename__ = 'projects'
__table_args__ = (
db.Index('release_name_idx', 'name'),
db.Index('release_name_is_active_idx', 'name', 'is_active'),
)
def __str__(self):
return self.name
def __repr__(self):
return '<Project %s: %s (%s)>' % (self.id, self.name, self.id)
@aggregated('uploads', db.Column(db.SmallInteger))
def uploads_count(self):
return db.func.count('1')
@property
def current_user_is_member(self):
if not current_user:
return False
elif not current_user.is_authenticated:
return False
elif current_user_is_roadie():
return True
else:
return current_user.id in self.member_ids
@property
def member_ids(self):
return [member.user.id for member in self.membership.all()]
@property
def pypi_json_url(self):
return f'https://pypi.org/pypi/{self.normalized_name}/json' # noqa
def file_upload(request):
# Before we do anything, if there isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(
HTTPForbidden,
"Invalid or non-existent authentication information.",
)
# distutils "helpfully" substitutes unknown, but "required" values with the
# string "UNKNOWN". This is basically never what anyone actually wants so
# we'll just go ahead and delete anything whose value is UNKNOWN.
for key in list(request.POST):
if request.POST.get(key) == "UNKNOWN":
del request.POST[key]
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
form.classifiers.choices = [
(c.classifier, c.classifier) for c in all_classifiers
]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
raise _exc_with_message(
HTTPBadRequest,
"{field}: {msgs[0]}".format(
field=field_name,
msgs=form.errors[field_name],
),
)
# TODO: We need a better method of blocking names rather than just
# hardcoding some names into source control.
if form.name.data.lower() in {"requirements.txt", "rrequirements.txt"}:
raise _exc_with_message(
HTTPBadRequest,
"The name {!r} is not allowed.".format(form.name.data),
)
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(
HTTPBadRequest,
"Upload payload does not have a file.",
)
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (
request.db.query(Project)
.filter(
Project.normalized_name ==
func.normalize_pep426_name(form.name.data)).one()
)
except NoResultFound:
# The project doesn't exist in our database, so we'll add it along with
# a role setting the current user as the "Owner" of the project.
project = Project(name=form.name.data)
request.db.add(project)
request.db.add(
Role(user=request.user, project=project, role_name="Owner")
)
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(
HTTPForbidden,
"You are not allowed to upload to {!r}.".format(project.name)
)
try:
release = (
request.db.query(Release)
.filter(
(Release.project == project) &
(Release.version == form.version.data)).one()
)
except NoResultFound:
release = Release(
project=project,
_classifiers=[
c for c in all_classifiers
if c.classifier in form.classifiers.data
],
dependencies=list(_construct_dependencies(
form,
{
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
}
)),
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary", "description", "license",
"author", "author_email", "maintainer", "maintainer_email",
"keywords", "platform",
"home_page", "download_url",
"requires_python",
}
}
)
request.db.add(release)
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = (
request.db.query(Release)
.filter(Release.project == project)
.all()
)
for i, r in enumerate(sorted(
releases, key=lambda x: packaging.version.parse(x.version))):
r._pypi_ordering = i
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain any path separators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(
HTTPBadRequest,
"Cannot upload a file with '/' or '\\' in the name.",
)
# Make sure the filename ends with an allowed extension.
if _dist_file_re.search(filename) is None:
raise _exc_with_message(HTTPBadRequest, "Invalid file extension.")
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest,
"The filename for {!r} must start with {!r}.".format(
project.name,
prefix,
)
)
# Check to see if the file that was uploaded exists already or not.
if request.db.query(
request.db.query(File)
.filter(File.filename == filename)
.exists()).scalar():
raise _exc_with_message(HTTPBadRequest, "File already exists.")
# Check to see if the file that was uploaded exists in our filename log.
if (request.db.query(
request.db.query(Filename)
.filter(Filename.filename == filename)
.exists()).scalar()):
raise _exc_with_message(
HTTPBadRequest,
"This filename has previously been used, you should use a "
"different version.",
)
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(os.path.join(tmpdir, filename), "wb") as fp:
file_size = 0
file_hash = hashlib.md5()
for chunk in iter(
lambda: request.POST["content"].file.read(8096), b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(HTTPBadRequest, "File too large.")
fp.write(chunk)
file_hash.update(chunk)
# Actually verify that the md5 hash of the file matches the expected
# md5 hash. We probably don't actually need to use hmac.compare_digest
# here since both the md5_digest and the file whose file_hash we've
# computed comes from the remote user, however better safe than sorry.
if not hmac.compare_digest(
form.md5_digest.data, file_hash.hexdigest()):
raise _exc_with_message(
HTTPBadRequest,
"The MD5 digest supplied does not match a digest calculated "
"from the uploaded file."
)
# TODO: Check the file to make sure it is a valid distribution file.
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
if set(plats) - ALLOWED_PLATFORMS:
raise _exc_with_message(
HTTPBadRequest,
"Binary wheel for an unsupported platform.",
)
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(
lambda: request.POST["gpg_signature"].file.read(8096),
b""):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(
HTTPBadRequest,
"Signature too large.",
)
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(
HTTPBadRequest,
"PGP signature is not ASCII armored.",
)
else:
has_signature = False
# TODO: We need some sort of trigger that will automatically add
# filenames to Filename instead of relying on this code running
# inside of our upload API.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=form.md5_digest.data,
)
request.db.add(file_)
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# committed.
storage = request.find_service(IFileStorage)
storage.store(file_.path, os.path.join(tmpdir, filename))
if has_signature:
storage.store(
file_.pgp_path,
os.path.join(tmpdir, filename + ".asc"),
)
return Response()
class Project(SitemapMixin, db.ModelBase):
__tablename__ = "packages"
__table_args__ = (CheckConstraint(
"name ~* '^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$'::text",
name="packages_valid_name",
), )
__repr__ = make_repr("name")
name = Column(Text, primary_key=True, nullable=False)
normalized_name = orm.column_property(func.normalize_pep426_name(name))
stable_version = Column(Text)
autohide = Column(Boolean, server_default=sql.true())
comments = Column(Boolean, server_default=sql.true())
bugtrack_url = Column(Text)
hosting_mode = Column(Text, nullable=False, server_default="pypi-only")
created = Column(
DateTime(timezone=False),
nullable=False,
server_default=sql.func.now(),
)
has_docs = Column(Boolean)
upload_limit = Column(Integer, nullable=True)
last_serial = Column(Integer, nullable=False, server_default=sql.text("0"))
allow_legacy_files = Column(
Boolean,
nullable=False,
server_default=sql.false(),
)
users = orm.relationship(
User,
secondary=Role.__table__,
backref="projects",
)
releases = orm.relationship(
"Release",
backref="project",
cascade="all, delete-orphan",
order_by=lambda: Release._pypi_ordering.desc(),
)
def __getitem__(self, version):
session = orm.object_session(self)
try:
return (session.query(Release).filter((Release.project == self) & (
Release.version == version)).one())
except NoResultFound:
raise KeyError from None
def __acl__(self):
session = orm.object_session(self)
acls = []
# Get all of the users for this project.
query = session.query(Role).filter(Role.project == self)
query = query.options(orm.lazyload("project"))
query = query.options(orm.joinedload("user").lazyload("emails"))
for role in sorted(
query.all(),
key=lambda x: ["Owner", "Maintainer"].index(x.role_name)):
acls.append((Allow, role.user.id, ["upload"]))
return acls
@property
def documentation_url(self):
# TODO: Move this into the database and elimnate the use of the
# threadlocal here.
request = get_current_request()
# If the project doesn't have docs, then we'll just return a None here.
if not self.has_docs:
return
return request.route_url("legacy.docs", project=self.name)
def file_upload(request):
# Before we do anything, if there isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(
HTTPForbidden,
"Invalid or non-existent authentication information.",
)
# distutils "helpfully" substitutes unknown, but "required" values with the
# string "UNKNOWN". This is basically never what anyone actually wants so
# we'll just go ahead and delete anything whose value is UNKNOWN.
for key in list(request.POST):
if request.POST.get(key) == "UNKNOWN":
del request.POST[key]
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Check if any fields were supplied as a tuple and have become a
# FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a
# FieldStorage, however.
# ref: https://github.com/pypa/warehouse/issues/2185
# ref: https://github.com/pypa/warehouse/issues/2491
for field in set(request.POST) - {'content', 'gpg_signature'}:
values = request.POST.getall(field)
if any(isinstance(value, FieldStorage) for value in values):
raise _exc_with_message(
HTTPBadRequest,
f"{field}: Should not be a tuple.",
)
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
form.classifiers.choices = [(c.classifier, c.classifier)
for c in all_classifiers]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
if field_name in form:
if form[field_name].description:
error_message = (
"{value!r} is an invalid value for {field}. ".format(
value=form[field_name].data,
field=form[field_name].description) +
"Error: {} ".format(form.errors[field_name][0]) + "see "
"https://packaging.python.org/specifications/core-metadata"
)
else:
error_message = "{field}: {msgs[0]}".format(
field=field_name,
msgs=form.errors[field_name],
)
else:
error_message = "Error: {}".format(form.errors[field_name][0])
raise _exc_with_message(
HTTPBadRequest,
error_message,
)
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(
HTTPBadRequest,
"Upload payload does not have a file.",
)
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
form.name.data)).one())
except NoResultFound:
# Check for AdminFlag set by a PyPI Administrator disabling new project
# registration, reasons for this include Spammers, security
# vulnerabilities, or just wanting to be lazy and not worry ;)
if AdminFlag.is_enabled(request.db,
'disallow-new-project-registration'):
raise _exc_with_message(
HTTPForbidden,
("New Project Registration Temporarily Disabled "
"See https://pypi.org/help#admin-intervention for details"),
) from None
# Ensure that user has at least one verified email address. This should
# reduce the ease of spam account creation and activity.
# TODO: Once legacy is shutdown consider the condition here, perhaps
# move to user.is_active or some other boolean
if not any(email.verified for email in request.user.emails):
raise _exc_with_message(
HTTPBadRequest,
("User {!r} has no verified email addresses, please verify "
"at least one address before registering a new project on "
"PyPI. See https://pypi.org/help/#verified-email "
"for more information.").format(request.user.username),
) from None
# Before we create the project, we're going to check our blacklist to
# see if this project is even allowed to be registered. If it is not,
# then we're going to deny the request to create this project.
if request.db.query(exists().where(
BlacklistedProject.name == func.normalize_pep426_name(
form.name.data))).scalar():
raise _exc_with_message(
HTTPBadRequest,
("The name {!r} is not allowed. "
"See https://pypi.org/help/#project-name "
"for more information.").format(form.name.data),
) from None
# Also check for collisions with Python Standard Library modules.
if (packaging.utils.canonicalize_name(form.name.data)
in STDLIB_PROHIBITTED):
raise _exc_with_message(
HTTPBadRequest,
("The name {!r} is not allowed (conflict with Python "
"Standard Library module name). See "
"https://pypi.org/help/#project-name for more information."
).format(form.name.data),
) from None
# The project doesn't exist in our database, so we'll add it along with
# a role setting the current user as the "Owner" of the project.
project = Project(name=form.name.data)
request.db.add(project)
request.db.add(
Role(user=request.user, project=project, role_name="Owner"))
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=project.name,
action="create",
submitted_by=request.user,
submitted_from=request.remote_addr,
), )
request.db.add(
JournalEntry(
name=project.name,
action="add Owner {}".format(request.user.username),
submitted_by=request.user,
submitted_from=request.remote_addr,
), )
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(
HTTPForbidden,
("The user '{0}' is not allowed to upload to project '{1}'. "
"See https://pypi.org/help#project-name for more information."
).format(request.user.username, project.name))
try:
canonical_version = packaging.utils.canonicalize_version(
form.version.data)
release = (request.db.query(Release).filter(
(Release.project == project)
& (Release.canonical_version == canonical_version)).one())
except MultipleResultsFound:
# There are multiple releases of this project which have the same
# canonical version that were uploaded before we checked for
# canonical version equivalence, so return the exact match instead
release = (request.db.query(
Release).filter((Release.project == project)
& (Release.version == form.version.data)).one())
except NoResultFound:
release = Release(
project=project,
_classifiers=[
c for c in all_classifiers
if c.classifier in form.classifiers.data
],
_pypi_hidden=False,
dependencies=list(
_construct_dependencies(
form, {
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
})),
canonical_version=canonical_version,
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary",
"description",
"license",
"author",
"author_email",
"maintainer",
"maintainer_email",
"keywords",
"platform",
"home_page",
"download_url",
"requires_python",
}
})
request.db.add(release)
# TODO: This should be handled by some sort of database trigger or
# a SQLAlchemy hook or the like instead of doing it inline in
# this view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="new release",
submitted_by=request.user,
submitted_from=request.remote_addr,
), )
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = (request.db.query(Release).filter(
Release.project == project).all())
for i, r in enumerate(
sorted(releases,
key=lambda x: packaging.version.parse(x.version))):
r._pypi_ordering = i
# TODO: Again, we should figure out a better solution to doing this than
# just inlining this inside this method.
if project.autohide:
for r in releases:
r._pypi_hidden = bool(not r == release)
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain any path separators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(
HTTPBadRequest,
"Cannot upload a file with '/' or '\\' in the name.",
)
# Make sure the filename ends with an allowed extension.
if _dist_file_regexes[project.allow_legacy_files].search(filename) is None:
raise _exc_with_message(
HTTPBadRequest,
"Invalid file extension. PEP 527 requires one of: .egg, .tar.gz, "
".whl, .zip (https://www.python.org/dev/peps/pep-0527/).")
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest,
"The filename for {!r} must start with {!r}.".format(
project.name,
prefix,
))
# Check the content type of what is being uploaded
if (not request.POST["content"].type
or request.POST["content"].type.startswith("image/")):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Ensure that the package filetpye is allowed.
# TODO: Once PEP 527 is completely implemented we should be able to delete
# this and just move it into the form itself.
if (not project.allow_legacy_files and form.filetype.data
not in {"sdist", "bdist_wheel", "bdist_egg"}):
raise _exc_with_message(HTTPBadRequest, "Unknown type of file.")
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
temporary_filename = os.path.join(tmpdir, filename)
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(temporary_filename, "wb") as fp:
file_size = 0
file_hashes = {
"md5": hashlib.md5(),
"sha256": hashlib.sha256(),
"blake2_256": hashlib.blake2b(digest_size=256 // 8),
}
for chunk in iter(lambda: request.POST["content"].file.read(8096),
b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(
HTTPBadRequest, "File too large. " +
"Limit for project {name!r} is {limit}MB".format(
name=project.name,
limit=file_size_limit // (1024 * 1024),
))
fp.write(chunk)
for hasher in file_hashes.values():
hasher.update(chunk)
# Take our hash functions and compute the final hashes for them now.
file_hashes = {
k: h.hexdigest().lower()
for k, h in file_hashes.items()
}
# Actually verify the digests that we've gotten. We're going to use
# hmac.compare_digest even though we probably don't actually need to
# because it's better safe than sorry. In the case of multiple digests
# we expect them all to be given.
if not all([
hmac.compare_digest(
getattr(form,
"{}_digest".format(digest_name)).data.lower(),
digest_value,
) for digest_name, digest_value in file_hashes.items()
if getattr(form, "{}_digest".format(digest_name)).data
]):
raise _exc_with_message(
HTTPBadRequest,
"The digest supplied does not match a digest calculated "
"from the uploaded file.")
# Check to see if the file that was uploaded exists already or not.
is_duplicate = _is_duplicate_file(request.db, filename, file_hashes)
if is_duplicate:
return Response()
elif is_duplicate is not None:
raise _exc_with_message(
HTTPBadRequest, "File already exists. "
"See " + request.route_url('help', _anchor='file-name-reuse'))
# Check to see if the file that was uploaded exists in our filename log
if (request.db.query(
request.db.query(Filename).filter(
Filename.filename == filename).exists()).scalar()):
raise _exc_with_message(
HTTPBadRequest,
"This filename has previously been used, you should use a "
"different version. "
"See " + request.route_url('help', _anchor='file-name-reuse'),
)
# Check to see if uploading this file would create a duplicate sdist
# for the current release.
if (form.filetype.data == "sdist" and request.db.query(
request.db.query(File).filter((File.release == release) & (
File.packagetype == "sdist")).exists()).scalar()):
raise _exc_with_message(
HTTPBadRequest,
"Only one sdist may be uploaded per release.",
)
# Check the file to make sure it is a valid distribution file.
if not _is_valid_dist_file(temporary_filename, form.filetype.data):
raise _exc_with_message(
HTTPBadRequest,
"Invalid distribution file.",
)
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
for plat in plats:
if not _valid_platform_tag(plat):
raise _exc_with_message(
HTTPBadRequest,
"Binary wheel '{filename}' has an unsupported "
"platform tag '{plat}'.".format(filename=filename,
plat=plat))
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(
lambda: request.POST["gpg_signature"].file.read(8096),
b""):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(
HTTPBadRequest,
"Signature too large.",
)
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(
HTTPBadRequest,
"PGP signature is not ASCII armored.",
)
else:
has_signature = False
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=file_hashes["md5"],
sha256_digest=file_hashes["sha256"],
blake2_256_digest=file_hashes["blake2_256"],
# Figure out what our filepath is going to be, we're going to use a
# directory structure based on the hash of the file contents. This
# will ensure that the contents of the file cannot change without
# it also changing the path that the file is saved too.
path="/".join([
file_hashes[PATH_HASHER][:2],
file_hashes[PATH_HASHER][2:4],
file_hashes[PATH_HASHER][4:],
filename,
]),
)
request.db.add(file_)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="add {python_version} file {filename}".format(
python_version=file_.python_version,
filename=file_.filename,
),
submitted_by=request.user,
submitted_from=request.remote_addr,
), )
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# committed.
storage = request.find_service(IFileStorage)
storage.store(
file_.path,
os.path.join(tmpdir, filename),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
if has_signature:
storage.store(
file_.pgp_path,
os.path.join(tmpdir, filename + ".asc"),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
# TODO: Once we no longer have the legacy code base running PyPI we can
# go ahead and delete this tiny bit of shim code, since it only
# exists to purge stuff on legacy PyPI when uploaded to Warehouse
old_domain = request.registry.settings.get("warehouse.legacy_domain")
if old_domain:
request.tm.get().addAfterCommitHook(
_legacy_purge,
args=["https://{}/pypi".format(old_domain)],
kws={"data": {
":action": "purge",
"project": project.name
}},
)
return Response()
def release_data(request, package_name, version):
try:
release = (request.db.query(Release).join(Project).filter((
Project.normalized_name == func.normalize_pep426_name(package_name)
) & (Release.version == version)).one())
except NoResultFound:
return {}
stats_svc = request.find_service(IDownloadStatService)
return {
"name":
release.project.name,
"version":
release.version,
"stable_version":
release.project.stable_version,
"bugtrack_url":
release.project.bugtrack_url,
"package_url":
request.route_url(
"packaging.project",
name=release.project.name,
),
"release_url":
request.route_url(
"packaging.release",
name=release.project.name,
version=release.version,
),
"docs_url":
release.project.documentation_url,
"home_page":
release.home_page,
"download_url":
release.download_url,
"project_url":
list(release.project_urls),
"author":
release.author,
"author_email":
release.author_email,
"maintainer":
release.maintainer,
"maintainer_email":
release.maintainer_email,
"summary":
release.summary,
"description":
release.description,
"license":
release.license,
"keywords":
release.keywords,
"platform":
release.platform,
"classifiers":
list(release.classifiers),
"requires":
list(release.requires),
"requires_dist":
list(release.requires_dist),
"provides":
list(release.provides),
"provides_dist":
list(release.provides_dist),
"obsoletes":
list(release.obsoletes),
"obsoletes_dist":
list(release.obsoletes_dist),
"requires_python":
release.requires_python,
"requires_external":
list(release.requires_external),
"_pypi_ordering":
release._pypi_ordering,
"_pypi_hidden":
release._pypi_hidden,
"downloads": {
"last_day": stats_svc.get_daily_stats(release.project.name),
"last_week": stats_svc.get_weekly_stats(release.project.name),
"last_month": stats_svc.get_monthly_stats(release.project.name),
},
}
class Project(db.Model, Syncable):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text, nullable=False, index=True)
normalized_name = orm.column_property(func.normalize_pep426_name(name))
description = db.Column(db.Text)
html_url = db.Column(db.String(255))
subscribers_count = db.Column(db.SmallInteger, default=0, nullable=False)
stargazers_count = db.Column(db.SmallInteger, default=0, nullable=False)
forks_count = db.Column(db.SmallInteger, default=0, nullable=False)
open_issues_count = db.Column(db.SmallInteger, default=0, nullable=False)
is_active = db.Column(db.Boolean, default=True, nullable=False, index=True)
transfer_issue_url = db.Column(db.String(255))
membership = db.relationship("ProjectMembership",
backref="project",
lazy="dynamic")
credentials = db.relationship("ProjectCredential",
backref="project",
lazy="dynamic")
uploads = db.relationship(
"ProjectUpload",
backref="project",
lazy="dynamic",
order_by=lambda: ProjectUpload.ordering.desc().nullslast(),
)
created_at = db.Column(db.DateTime, nullable=True)
updated_at = db.Column(db.DateTime, nullable=True)
pushed_at = db.Column(db.DateTime, nullable=True)
__tablename__ = "projects"
__table_args__ = (
db.Index("release_name_idx", "name"),
db.Index("release_name_is_active_idx", "name", "is_active"),
)
def __str__(self):
return self.name
@aggregated("uploads", db.Column(db.SmallInteger))
def uploads_count(self):
return db.func.count("1")
@property
def current_user_is_member(self):
if not current_user:
return False
elif not current_user.is_authenticated:
return False
elif current_user_is_roadie():
return True
else:
return current_user.id in self.member_ids
@property
def member_ids(self):
return [member.user.id for member in self.membership.all()]
@property
def leads(self):
leads = self.membership.filter(
ProjectMembership.is_lead.is_(True),
ProjectMembership.user_id.in_(User.active_members().options(
orm.load_only("id"))),
)
return [member.user for member in leads]
@property
def pypi_json_url(self):
return f"https://pypi.org/pypi/{self.normalized_name}/json" # noqa
def file_upload(request):
# Before we do anything, if there isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(
HTTPForbidden,
"Invalid or non-existent authentication information.",
)
# distutils "helpfully" substitutes unknown, but "required" values with the
# string "UNKNOWN". This is basically never what anyone actually wants so
# we'll just go ahead and delete anything whose value is UNKNOWN.
for key in list(request.POST):
if request.POST.get(key) == "UNKNOWN":
del request.POST[key]
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
form.classifiers.choices = [
(c.classifier, c.classifier) for c in all_classifiers
]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
raise _exc_with_message(
HTTPBadRequest,
"{field}: {msgs[0]}".format(
field=field_name,
msgs=form.errors[field_name],
),
)
# TODO: We need a better method of blocking names rather than just
# hardcoding some names into source control.
if form.name.data.lower() in {"requirements.txt", "rrequirements.txt"}:
raise _exc_with_message(
HTTPBadRequest,
"The name {!r} is not allowed.".format(form.name.data),
)
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(
HTTPBadRequest,
"Upload payload does not have a file.",
)
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (
request.db.query(Project)
.filter(
Project.normalized_name ==
func.normalize_pep426_name(form.name.data)).one()
)
except NoResultFound:
# The project doesn't exist in our database, so we'll add it along with
# a role setting the current user as the "Owner" of the project.
project = Project(name=form.name.data)
request.db.add(project)
request.db.add(
Role(user=request.user, project=project, role_name="Owner")
)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=project.name,
action="create",
submitted_by=request.user,
submitted_from=request.client_addr,
),
)
request.db.add(
JournalEntry(
name=project.name,
action="add Owner {}".format(request.user.username),
submitted_by=request.user,
submitted_from=request.client_addr,
),
)
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(
HTTPForbidden,
"You are not allowed to upload to {!r}.".format(project.name)
)
try:
release = (
request.db.query(Release)
.filter(
(Release.project == project) &
(Release.version == form.version.data)).one()
)
except NoResultFound:
release = Release(
project=project,
_classifiers=[
c for c in all_classifiers
if c.classifier in form.classifiers.data
],
_pypi_hidden=False,
dependencies=list(_construct_dependencies(
form,
{
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
}
)),
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary", "description", "license",
"author", "author_email", "maintainer", "maintainer_email",
"keywords", "platform",
"home_page", "download_url",
"requires_python",
}
}
)
request.db.add(release)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="new release",
submitted_by=request.user,
submitted_from=request.client_addr,
),
)
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = (
request.db.query(Release)
.filter(Release.project == project)
.all()
)
for i, r in enumerate(sorted(
releases, key=lambda x: packaging.version.parse(x.version))):
r._pypi_ordering = i
# TODO: Again, we should figure out a better solution to doing this than
# just inlining this inside this method.
if project.autohide:
for r in releases:
r._pypi_hidden = bool(not r == release)
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain any path separators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(
HTTPBadRequest,
"Cannot upload a file with '/' or '\\' in the name.",
)
# Make sure the filename ends with an allowed extension.
if _dist_file_re.search(filename) is None:
raise _exc_with_message(HTTPBadRequest, "Invalid file extension.")
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest,
"The filename for {!r} must start with {!r}.".format(
project.name,
prefix,
)
)
# Check the content type of what is being uploaded
if (not request.POST["content"].type or
request.POST["content"].type.startswith("image/")):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Check to see if the file that was uploaded exists already or not.
if request.db.query(
request.db.query(File)
.filter(File.filename == filename)
.exists()).scalar():
raise _exc_with_message(HTTPBadRequest, "File already exists.")
# Check to see if the file that was uploaded exists in our filename log.
if (request.db.query(
request.db.query(Filename)
.filter(Filename.filename == filename)
.exists()).scalar()):
raise _exc_with_message(
HTTPBadRequest,
"This filename has previously been used, you should use a "
"different version.",
)
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
temporary_filename = os.path.join(tmpdir, filename)
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(temporary_filename, "wb") as fp:
file_size = 0
file_hashes = {
"md5": hashlib.md5(),
"sha256": hashlib.sha256(),
"blake2_256": blake2b(digest_size=256 // 8),
}
for chunk in iter(
lambda: request.POST["content"].file.read(8096), b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(HTTPBadRequest, "File too large.")
fp.write(chunk)
for hasher in file_hashes.values():
hasher.update(chunk)
# Take our hash functions and compute the final hashes for them now.
file_hashes = {
k: h.hexdigest().lower()
for k, h in file_hashes.items()
}
# Actually verify the digests that we've gotten. We're going to use
# hmac.compare_digest even though we probably don't actually need to
# because it's better safe than sorry. In the case of multiple digests
# we expect them all to be given.
if not all([
hmac.compare_digest(
getattr(form, "{}_digest".format(digest_name)).data.lower(),
digest_value,
)
for digest_name, digest_value in file_hashes.items()
if getattr(form, "{}_digest".format(digest_name)).data
]):
raise _exc_with_message(
HTTPBadRequest,
"The digest supplied does not match a digest calculated "
"from the uploaded file."
)
# Check the file to make sure it is a valid distribution file.
if not _is_valid_dist_file(temporary_filename, form.filetype.data):
raise _exc_with_message(
HTTPBadRequest,
"Invalid distribution file.",
)
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
for plat in plats:
if not _valid_platform_tag(plat):
raise _exc_with_message(
HTTPBadRequest,
"Binary wheel '{filename}' has an unsupported "
"platform tag '{plat}'."
.format(filename=filename, plat=plat)
)
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(
lambda: request.POST["gpg_signature"].file.read(8096),
b""):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(
HTTPBadRequest,
"Signature too large.",
)
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(
HTTPBadRequest,
"PGP signature is not ASCII armored.",
)
else:
has_signature = False
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=file_hashes["md5"],
sha256_digest=file_hashes["sha256"],
blake2_256_digest=file_hashes["blake2_256"],
# Figure out what our filepath is going to be, we're going to use a
# directory structure based on the hash of the file contents. This
# will ensure that the contents of the file cannot change without
# it also changing the path that the file is saved too.
path="/".join([
file_hashes[PATH_HASHER][:2],
file_hashes[PATH_HASHER][2:4],
file_hashes[PATH_HASHER][4:],
filename,
]),
)
request.db.add(file_)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="add {python_version} file {filename}".format(
python_version=file_.python_version,
filename=file_.filename,
),
submitted_by=request.user,
submitted_from=request.client_addr,
),
)
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# committed.
storage = request.find_service(IFileStorage)
storage.store(
file_.path,
os.path.join(tmpdir, filename),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
if has_signature:
storage.store(
file_.pgp_path,
os.path.join(tmpdir, filename + ".asc"),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
# TODO: Once we no longer have the legacy code base running PyPI we can
# go ahead and delete this tiny bit of shim code, since it only
# exists to purge stuff on legacy PyPI when uploaded to Warehouse
old_domain = request.registry.settings.get("warehouse.legacy_domain")
if old_domain:
request.tm.get().addAfterCommitHook(
_legacy_purge,
args=["https://{}/pypi".format(old_domain)],
kws={"data": {":action": "purge", "project": project.name}},
)
return Response()
class Project(SitemapMixin, db.Model):
__tablename__ = "projects"
__table_args__ = (CheckConstraint(
"name ~* '^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$'::text",
name="projects_valid_name",
), )
__repr__ = make_repr("name")
name = Column(Text, nullable=False)
normalized_name = orm.column_property(func.normalize_pep426_name(name))
created = Column(
DateTime(timezone=False),
nullable=False,
server_default=sql.func.now(),
index=True,
)
has_docs = Column(Boolean)
upload_limit = Column(Integer, nullable=True)
last_serial = Column(Integer, nullable=False, server_default=sql.text("0"))
zscore = Column(Float, nullable=True)
total_size = Column(BigInteger, server_default=sql.text("0"))
users = orm.relationship(User,
secondary=Role.__table__,
backref="projects")
releases = orm.relationship(
"Release",
backref="project",
cascade="all, delete-orphan",
order_by=lambda: Release._pypi_ordering.desc(),
passive_deletes=True,
)
events = orm.relationship("ProjectEvent",
backref="project",
cascade="all, delete-orphan",
lazy=True)
def __getitem__(self, version):
session = orm.object_session(self)
canonical_version = packaging.utils.canonicalize_version(version)
try:
return (session.query(Release).filter(
Release.project == self,
Release.canonical_version == canonical_version,
).one())
except MultipleResultsFound:
# There are multiple releases of this project which have the same
# canonical version that were uploaded before we checked for
# canonical version equivalence, so return the exact match instead
try:
return (session.query(Release).filter(
Release.project == self, Release.version == version).one())
except NoResultFound:
# There are multiple releases of this project which have the
# same canonical version, but none that have the exact version
# specified, so just 404
raise KeyError from None
except NoResultFound:
raise KeyError from None
def __acl__(self):
session = orm.object_session(self)
acls = [
(Allow, "group:admins", "admin"),
(Allow, "group:moderators", "moderator"),
]
# Get all of the users for this project.
query = session.query(Role).filter(Role.project == self)
query = query.options(orm.lazyload("project"))
query = query.options(orm.joinedload("user").lazyload("emails"))
query = query.join(User).order_by(User.id.asc())
for role in sorted(
query.all(),
key=lambda x: ["Owner", "Maintainer"].index(x.role_name)):
if role.role_name == "Owner":
acls.append(
(Allow, str(role.user.id), ["manage:project", "upload"]))
else:
acls.append((Allow, str(role.user.id), ["upload"]))
return acls
def record_event(self, *, tag, ip_address, additional=None):
session = orm.object_session(self)
event = ProjectEvent(project=self,
tag=tag,
ip_address=ip_address,
additional=additional)
session.add(event)
session.flush()
return event
@property
def documentation_url(self):
# TODO: Move this into the database and eliminate the use of the
# threadlocal here.
request = get_current_request()
# If the project doesn't have docs, then we'll just return a None here.
if not self.has_docs:
return
return request.route_url("legacy.docs", project=self.name)
@property
def all_versions(self):
return (orm.object_session(self).query(
Release.version, Release.created, Release.is_prerelease,
Release.yanked).filter(Release.project == self).order_by(
Release._pypi_ordering.desc()).all())
@property
def latest_version(self):
return (orm.object_session(self).query(
Release.version, Release.created, Release.is_prerelease).filter(
Release.project == self, Release.yanked.is_(False)).order_by(
Release.is_prerelease.nullslast(),
Release._pypi_ordering.desc()).first())
def add_blacklist(request):
project_name = request.POST.get("project")
if project_name is None:
raise HTTPBadRequest("Must have a project to confirm.")
comment = request.POST.get("comment", "")
# Verify that the user has confirmed the request to blacklist.
confirm = request.POST.get("confirm")
if not confirm:
request.session.flash(
"Must confirm the blacklist request.",
queue="error",
)
return HTTPSeeOther(request.current_route_path())
elif canonicalize_name(confirm) != canonicalize_name(project_name):
request.session.flash(
f"{confirm!r} is not the same as {project_name!r}",
queue="error",
)
return HTTPSeeOther(request.current_route_path())
# Add our requested blacklist.
request.db.add(
BlacklistedProject(
name=project_name,
comment=comment,
blacklisted_by=request.user,
))
# Go through and delete anything that we need to delete so that our
# blacklist actually blocks things and isn't ignored (since the blacklist
# only takes effect on new project registration).
# TODO: This should be in a generic function somewhere instead of putting
# it here, however this will have to do for now.
# TODO: We don't actually delete files from the data store. We should add
# some kind of garbage collection at some point.
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
project_name)).first())
if project is not None:
request.db.add(
JournalEntry(
name=project.name,
action="remove",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.db.query(Role).filter(Role.project == project).delete()
request.db.query(File).filter(File.name == project.name).delete()
(request.db.query(Dependency).filter(
Dependency.name == project.name).delete())
(request.db.execute(release_classifiers.delete().where(
release_classifiers.c.name == project.name)))
request.db.query(Release).filter(Release.name == project.name).delete()
request.db.query(Project).filter(Project.name == project.name).delete()
request.session.flash(
f"Successfully blacklisted {project_name!r}",
queue="success",
)
return HTTPSeeOther(request.route_path("admin.blacklist.list"))
class Organization(HasEvents, db.Model):
__tablename__ = "organizations"
__table_args__ = (CheckConstraint(
"name ~* '^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$'::text",
name="organizations_valid_name",
), )
__repr__ = make_repr("name")
name = Column(Text, nullable=False)
normalized_name = orm.column_property(func.normalize_pep426_name(name))
display_name = Column(Text, nullable=False)
orgtype = Column(
Enum(OrganizationType, values_callable=lambda x: [e.value for e in x]),
nullable=False,
)
link_url = Column(URLType, nullable=False)
description = Column(Text, nullable=False)
is_active = Column(Boolean, nullable=False, server_default=sql.false())
is_approved = Column(Boolean)
created = Column(
DateTime(timezone=False),
nullable=False,
server_default=sql.func.now(),
index=True,
)
date_approved = Column(
DateTime(timezone=False),
nullable=True,
onupdate=func.now(),
)
users = orm.relationship(
User,
secondary=OrganizationRole.__table__,
backref="organizations" # type: ignore # noqa
)
projects = orm.relationship(
"Project",
secondary=OrganizationProject.__table__,
backref="organizations" # type: ignore # noqa
)
def record_event(self, *, tag, ip_address, additional={}):
"""Record organization name in events in case organization is ever deleted."""
super().record_event(
tag=tag,
ip_address=ip_address,
additional={
"organization_name": self.name,
**additional
},
)
def __acl__(self):
session = orm.object_session(self)
acls = [
(Allow, "group:admins", "admin"),
(Allow, "group:moderators", "moderator"),
]
# Get all of the users for this organization.
query = session.query(OrganizationRole).filter(
OrganizationRole.organization == self)
query = query.options(orm.lazyload("organization"))
query = query.join(User).order_by(User.id.asc())
for role in sorted(
query.all(),
key=lambda x: [e.value for e in OrganizationRoleType].index(
x.role_name),
):
# Allow all people in organization read access.
# Allow write access depending on role.
if role.role_name == OrganizationRoleType.Owner:
acls.append((
Allow,
f"user:{role.user.id}",
["view:organization", "manage:organization"],
))
elif role.role_name == OrganizationRoleType.BillingManager:
acls.append((
Allow,
f"user:{role.user.id}",
["view:organization", "manage:billing"],
))
elif role.role_name == OrganizationRoleType.Manager:
acls.append((
Allow,
f"user:{role.user.id}",
["view:organization", "manage:team"],
))
else:
# No member-specific write access needed for now.
acls.append(
(Allow, f"user:{role.user.id}", ["view:organization"]))
return acls
def file_upload(request):
# Before we do anything, if their isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(HTTPForbidden, "Invalid or non-existent authentication information.")
# distutils "helpfully" substitutes unknown, but "required" values with the
# string "UNKNOWN". This is basically never what anyone actually wants so
# we'll just go ahead and delete anything whose value is UNKNOWN.
for key in list(request.POST):
if request.POST.get(key) == "UNKNOWN":
del request.POST[key]
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
form.classifiers.choices = [(c.classifier, c.classifier) for c in all_classifiers]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
raise _exc_with_message(
HTTPBadRequest, "{field}: {msgs[0]}".format(field=field_name, msgs=form.errors[field_name])
)
# TODO: We need a better method of blocking names rather than jsut
# hardcoding some names into source control.
if form.name.data.lower() in {"requirements.txt", "rrequirements.txt"}:
raise _exc_with_message(HTTPBadRequest, "The name {!r} is not allowed.".format(form.name.data))
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(HTTPBadRequest, "Upload payload does not have a file.")
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (
request.db.query(Project)
.filter(Project.normalized_name == func.normalize_pep426_name(form.name.data))
.one()
)
except NoResultFound:
# The project doesn't exist in our database, so we'll add it along with
# a role setting the current user as the "Owner" of the project.
project = Project(name=form.name.data)
request.db.add(project)
request.db.add(Role(user=request.user, project=project, role_name="Owner"))
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(HTTPForbidden, "You are not allowed to upload to {!r}.".format(project.name))
try:
release = (
request.db.query(Release)
.filter((Release.project == project) & (Release.version == form.version.data))
.one()
)
except NoResultFound:
release = Release(
project=project,
_classifiers=[c for c in all_classifiers if c.classifier in form.classifiers.data],
dependencies=list(
_construct_dependencies(
form,
{
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
},
)
),
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary",
"description",
"license",
"author",
"author_email",
"maintainer",
"maintainer_email",
"keywords",
"platform",
"home_page",
"download_url",
"requires_python",
}
}
)
request.db.add(release)
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = request.db.query(Release).filter(Release.project == project).all()
for i, r in enumerate(sorted(releases, key=lambda x: packaging.version.parse(x.version))):
r._pypi_ordering = i
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain and path seperators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name.")
# Make sure the filename ends with an allowed extension.
if _dist_file_re.search(filename) is None:
raise _exc_with_message(HTTPBadRequest, "Invalid file extension.")
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest, "The filename for {!r} must start with {!r}.".format(project.name, prefix)
)
# Check to see if the file that was uploaded exists already or not.
if request.db.query(request.db.query(File).filter(File.filename == filename).exists()).scalar():
raise _exc_with_message(HTTPBadRequest, "File already exists.")
# Check to see if the file that was uploaded exists in our filename log.
if request.db.query(request.db.query(Filename).filter(Filename.filename == filename).exists()).scalar():
raise _exc_with_message(
HTTPBadRequest, "This filename has previously been used, you should use a " "different version."
)
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(os.path.join(tmpdir, filename), "wb") as fp:
file_size = 0
file_hash = hashlib.md5()
for chunk in iter(lambda: request.POST["content"].file.read(8096), b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(HTTPBadRequest, "File too large.")
fp.write(chunk)
file_hash.update(chunk)
# Actually verify that the md5 hash of the file matches the expected
# md5 hash. We probably don't actually need to use hmac.compare_digest
# here since both the md5_digest and the file whose file_hash we've
# computed comes from the remote user, however better safe than sorry.
if not hmac.compare_digest(form.md5_digest.data, file_hash.hexdigest()):
raise _exc_with_message(
HTTPBadRequest, "The MD5 digest supplied does not match a digest calculated " "from the uploaded file."
)
# TODO: Check the file to make sure it is a valid distribution file.
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
if set(plats) - ALLOWED_PLATFORMS:
raise _exc_with_message(HTTPBadRequest, "Binary wheel for an unsupported platform.")
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(lambda: request.POST["gpg_signature"].file.read(8096), b""):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(HTTPBadRequest, "Signature too large.")
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(HTTPBadRequest, "PGP signature is not ASCII armored.")
else:
has_signature = False
# TODO: We need some sort of trigger that will automatically add
# filenames to Filename instead of relying on this code running
# inside of our upload API.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=form.md5_digest.data,
)
request.db.add(file_)
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# commited.
storage = request.find_service(IFileStorage)
storage.store(file_.path, os.path.join(tmpdir, filename))
if has_signature:
storage.store(file_.pgp_path, os.path.join(tmpdir, filename + ".asc"))
return Response()
def file_upload(request):
# If we're in read-only mode, let upload clients know
if request.flags.enabled("read-only"):
raise _exc_with_message(
HTTPForbidden, "Read-only mode: Uploads are temporarily disabled")
# Log an attempt to upload
metrics = request.find_service(IMetricsService, context=None)
metrics.increment("warehouse.upload.attempt")
# Before we do anything, if there isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(
HTTPForbidden,
"Invalid or non-existent authentication information.")
# Ensure that user has a verified, primary email address. This should both
# reduce the ease of spam account creation and activity, as well as act as
# a forcing function for https://github.com/pypa/warehouse/issues/3632.
# TODO: Once https://github.com/pypa/warehouse/issues/3632 has been solved,
# we might consider a different condition, possibly looking at
# User.is_active instead.
if not (request.user.primary_email
and request.user.primary_email.verified):
raise _exc_with_message(
HTTPBadRequest,
("User {!r} does not have a verified primary email address. "
"Please add a verified primary email before attempting to "
"upload to PyPI. See {project_help} for more information."
"for more information.").format(
request.user.username,
project_help=request.help_url(_anchor="verified-email"),
),
) from None
# Do some cleanup of the various form fields
for key in list(request.POST):
value = request.POST.get(key)
if isinstance(value, str):
# distutils "helpfully" substitutes unknown, but "required" values
# with the string "UNKNOWN". This is basically never what anyone
# actually wants so we'll just go ahead and delete anything whose
# value is UNKNOWN.
if value.strip() == "UNKNOWN":
del request.POST[key]
# Escape NUL characters, which psycopg doesn't like
if "\x00" in value:
request.POST[key] = value.replace("\x00", "\\x00")
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Check if any fields were supplied as a tuple and have become a
# FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a
# FieldStorage, however.
# ref: https://github.com/pypa/warehouse/issues/2185
# ref: https://github.com/pypa/warehouse/issues/2491
for field in set(request.POST) - {"content", "gpg_signature"}:
values = request.POST.getall(field)
if any(isinstance(value, FieldStorage) for value in values):
raise _exc_with_message(HTTPBadRequest,
f"{field}: Should not be a tuple.")
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
# Add a validator for deprecated classifiers
form.classifiers.validators.append(_no_deprecated_classifiers(request))
form.classifiers.choices = [(c.classifier, c.classifier)
for c in all_classifiers]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
if field_name in form:
field = form[field_name]
if field.description and isinstance(field, wtforms.StringField):
error_message = (
"{value!r} is an invalid value for {field}. ".format(
value=field.data, field=field.description) +
"Error: {} ".format(form.errors[field_name][0]) + "See "
"https://packaging.python.org/specifications/core-metadata"
)
else:
error_message = "Invalid value for {field}. Error: {msgs[0]}".format(
field=field_name, msgs=form.errors[field_name])
else:
error_message = "Error: {}".format(form.errors[field_name][0])
raise _exc_with_message(HTTPBadRequest, error_message)
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(HTTPBadRequest,
"Upload payload does not have a file.")
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (request.db.query(Project).filter(
Project.normalized_name == func.normalize_pep426_name(
form.name.data)).one())
except NoResultFound:
# Check for AdminFlag set by a PyPI Administrator disabling new project
# registration, reasons for this include Spammers, security
# vulnerabilities, or just wanting to be lazy and not worry ;)
if request.flags.enabled("disallow-new-project-registration"):
raise _exc_with_message(
HTTPForbidden,
("New project registration temporarily disabled. "
"See {projecthelp} for details").format(
projecthelp=request.help_url(
_anchor="admin-intervention")),
) from None
# Before we create the project, we're going to check our blacklist to
# see if this project is even allowed to be registered. If it is not,
# then we're going to deny the request to create this project.
if request.db.query(exists().where(
BlacklistedProject.name == func.normalize_pep426_name(
form.name.data))).scalar():
raise _exc_with_message(
HTTPBadRequest,
("The name {name!r} isn't allowed. "
"See {projecthelp} "
"for more information.").format(
name=form.name.data,
projecthelp=request.help_url(_anchor="project-name"),
),
) from None
# Also check for collisions with Python Standard Library modules.
if packaging.utils.canonicalize_name(
form.name.data) in STDLIB_PROHIBITTED:
raise _exc_with_message(
HTTPBadRequest,
("The name {name!r} isn't allowed (conflict with Python "
"Standard Library module name). See "
"{projecthelp} for more information.").format(
name=form.name.data,
projecthelp=request.help_url(_anchor="project-name"),
),
) from None
# The project doesn't exist in our database, so first we'll check for
# projects with a similar name
squattees = (request.db.query(Project).filter(
func.levenshtein(Project.normalized_name,
func.normalize_pep426_name(form.name.data)) <= 2).
all())
# Next we'll create the project
project = Project(name=form.name.data)
request.db.add(project)
# Now that the project exists, add any squats which it is the squatter for
for squattee in squattees:
request.db.add(Squat(squatter=project, squattee=squattee))
# Then we'll add a role setting the current user as the "Owner" of the
# project.
request.db.add(
Role(user=request.user, project=project, role_name="Owner"))
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=project.name,
action="create",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
request.db.add(
JournalEntry(
name=project.name,
action="add Owner {}".format(request.user.username),
submitted_by=request.user,
submitted_from=request.remote_addr,
))
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(
HTTPForbidden,
("The credential associated with user '{0}' "
"isn't allowed to upload to project '{1}'. "
"See {2} for more information.").format(
request.user.username,
project.name,
request.help_url(_anchor="project-name"),
),
)
# Update name if it differs but is still equivalent. We don't need to check if
# they are equivalent when normalized because that's already been done when we
# queried for the project.
if project.name != form.name.data:
project.name = form.name.data
# Render our description so we can save from having to render this data every time
# we load a project description page.
rendered = None
if form.description.data:
description_content_type = form.description_content_type.data
if not description_content_type:
description_content_type = "text/x-rst"
rendered = readme.render(form.description.data,
description_content_type,
use_fallback=False)
# Uploading should prevent broken rendered descriptions.
if rendered is None:
if form.description_content_type.data:
message = (
"The description failed to render "
"for '{description_content_type}'.").format(
description_content_type=description_content_type)
else:
message = ("The description failed to render "
"in the default format of reStructuredText.")
raise _exc_with_message(
HTTPBadRequest,
"{message} See {projecthelp} for more information.".format(
message=message,
projecthelp=request.help_url(
_anchor="description-content-type"),
),
) from None
try:
canonical_version = packaging.utils.canonicalize_version(
form.version.data)
release = (request.db.query(Release).filter(
(Release.project == project)
& (Release.canonical_version == canonical_version)).one())
except MultipleResultsFound:
# There are multiple releases of this project which have the same
# canonical version that were uploaded before we checked for
# canonical version equivalence, so return the exact match instead
release = (request.db.query(
Release).filter((Release.project == project)
& (Release.version == form.version.data)).one())
except NoResultFound:
release = Release(
project=project,
_classifiers=[
c for c in all_classifiers
if c.classifier in form.classifiers.data
],
dependencies=list(
_construct_dependencies(
form,
{
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
},
)),
canonical_version=canonical_version,
description=Description(
content_type=form.description_content_type.data,
raw=form.description.data or "",
html=rendered or "",
rendered_by=readme.renderer_version(),
),
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary",
"license",
"author",
"author_email",
"maintainer",
"maintainer_email",
"keywords",
"platform",
"home_page",
"download_url",
"requires_python",
}
},
uploader=request.user,
uploaded_via=request.user_agent,
)
request.db.add(release)
# TODO: This should be handled by some sort of database trigger or
# a SQLAlchemy hook or the like instead of doing it inline in
# this view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="new release",
submitted_by=request.user,
submitted_from=request.remote_addr,
))
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = (request.db.query(Release).filter(
Release.project == project).options(
orm.load_only(Release._pypi_ordering)).all())
for i, r in enumerate(
sorted(releases,
key=lambda x: packaging.version.parse(x.version))):
r._pypi_ordering = i
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain any path separators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(
HTTPBadRequest,
"Cannot upload a file with '/' or '\\' in the name.")
# Make sure the filename ends with an allowed extension.
if _dist_file_regexes[project.allow_legacy_files].search(filename) is None:
raise _exc_with_message(
HTTPBadRequest,
"Invalid file extension: Use .egg, .tar.gz, .whl or .zip "
"extension. (https://www.python.org/dev/peps/pep-0527)",
)
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest,
"Start filename for {!r} with {!r}.".format(project.name, prefix),
)
# Check the content type of what is being uploaded
if not request.POST["content"].type or request.POST[
"content"].type.startswith("image/"):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Ensure that the package filetype is allowed.
# TODO: Once PEP 527 is completely implemented we should be able to delete
# this and just move it into the form itself.
if not project.allow_legacy_files and form.filetype.data not in {
"sdist",
"bdist_wheel",
"bdist_egg",
}:
raise _exc_with_message(HTTPBadRequest, "Unknown type of file.")
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
temporary_filename = os.path.join(tmpdir, filename)
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(temporary_filename, "wb") as fp:
file_size = 0
file_hashes = {
"md5": hashlib.md5(),
"sha256": hashlib.sha256(),
"blake2_256": hashlib.blake2b(digest_size=256 // 8),
}
for chunk in iter(lambda: request.POST["content"].file.read(8096),
b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(
HTTPBadRequest,
"File too large. " +
"Limit for project {name!r} is {limit} MB. ".format(
name=project.name,
limit=file_size_limit // (1024 * 1024)) + "See " +
request.help_url(_anchor="file-size-limit"),
)
fp.write(chunk)
for hasher in file_hashes.values():
hasher.update(chunk)
# Take our hash functions and compute the final hashes for them now.
file_hashes = {
k: h.hexdigest().lower()
for k, h in file_hashes.items()
}
# Actually verify the digests that we've gotten. We're going to use
# hmac.compare_digest even though we probably don't actually need to
# because it's better safe than sorry. In the case of multiple digests
# we expect them all to be given.
if not all([
hmac.compare_digest(
getattr(form,
"{}_digest".format(digest_name)).data.lower(),
digest_value,
) for digest_name, digest_value in file_hashes.items()
if getattr(form, "{}_digest".format(digest_name)).data
]):
raise _exc_with_message(
HTTPBadRequest,
"The digest supplied does not match a digest calculated "
"from the uploaded file.",
)
# Check to see if the file that was uploaded exists already or not.
is_duplicate = _is_duplicate_file(request.db, filename, file_hashes)
if is_duplicate:
return Response()
elif is_duplicate is not None:
raise _exc_with_message(
HTTPBadRequest,
# Note: Changing this error message to something that doesn't
# start with "File already exists" will break the
# --skip-existing functionality in twine
# ref: https://github.com/pypa/warehouse/issues/3482
# ref: https://github.com/pypa/twine/issues/332
"File already exists. See " +
request.help_url(_anchor="file-name-reuse"),
)
# Check to see if the file that was uploaded exists in our filename log
if request.db.query(
request.db.query(Filename).filter(
Filename.filename == filename).exists()).scalar():
raise _exc_with_message(
HTTPBadRequest,
"This filename has already been used, use a "
"different version. "
"See " + request.help_url(_anchor="file-name-reuse"),
)
# Check to see if uploading this file would create a duplicate sdist
# for the current release.
if (form.filetype.data == "sdist" and request.db.query(
request.db.query(File).filter((File.release == release) & (
File.packagetype == "sdist")).exists()).scalar()):
raise _exc_with_message(
HTTPBadRequest, "Only one sdist may be uploaded per release.")
# Check the file to make sure it is a valid distribution file.
if not _is_valid_dist_file(temporary_filename, form.filetype.data):
raise _exc_with_message(HTTPBadRequest,
"Invalid distribution file.")
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
for plat in plats:
if not _valid_platform_tag(plat):
raise _exc_with_message(
HTTPBadRequest,
"Binary wheel '{filename}' has an unsupported "
"platform tag '{plat}'.".format(filename=filename,
plat=plat),
)
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(
lambda: request.POST["gpg_signature"].file.read(8096),
b""):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(HTTPBadRequest,
"Signature too large.")
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(
HTTPBadRequest, "PGP signature isn't ASCII armored.")
else:
has_signature = False
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=file_hashes["md5"],
sha256_digest=file_hashes["sha256"],
blake2_256_digest=file_hashes["blake2_256"],
# Figure out what our filepath is going to be, we're going to use a
# directory structure based on the hash of the file contents. This
# will ensure that the contents of the file cannot change without
# it also changing the path that the file is saved too.
path="/".join([
file_hashes[PATH_HASHER][:2],
file_hashes[PATH_HASHER][2:4],
file_hashes[PATH_HASHER][4:],
filename,
]),
uploaded_via=request.user_agent,
)
request.db.add(file_)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="add {python_version} file {filename}".format(
python_version=file_.python_version,
filename=file_.filename),
submitted_by=request.user,
submitted_from=request.remote_addr,
))
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# committed.
storage = request.find_service(IFileStorage)
storage.store(
file_.path,
os.path.join(tmpdir, filename),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
if has_signature:
storage.store(
file_.pgp_path,
os.path.join(tmpdir, filename + ".asc"),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
# Log a successful upload
metrics.increment("warehouse.upload.ok",
tags=[f"filetype:{form.filetype.data}"])
return Response()
def file_upload(request):
# If we're in read-only mode, let upload clients know
if request.flags.enabled("read-only"):
raise _exc_with_message(
HTTPForbidden, "Read-only mode: Uploads are temporarily disabled"
)
# Before we do anything, if there isn't an authenticated user with this
# request, then we'll go ahead and bomb out.
if request.authenticated_userid is None:
raise _exc_with_message(
HTTPForbidden, "Invalid or non-existent authentication information."
)
# Ensure that user has a verified, primary email address. This should both
# reduce the ease of spam account creation and activty, as well as act as
# a forcing function for https://github.com/pypa/warehouse/issues/3632.
# TODO: Once https://github.com/pypa/warehouse/issues/3632 has been solved,
# we might consider a different condition, possibly looking at
# User.is_active instead.
if not (request.user.primary_email and request.user.primary_email.verified):
raise _exc_with_message(
HTTPBadRequest,
(
"User {!r} does not have a verified primary email address. "
"Please add a verified primary email before attempting to "
"upload to PyPI. See {project_help} for more information."
"for more information."
).format(
request.user.username,
project_help=request.help_url(_anchor="verified-email"),
),
) from None
# Do some cleanup of the various form fields
for key in list(request.POST):
value = request.POST.get(key)
if isinstance(value, str):
# distutils "helpfully" substitutes unknown, but "required" values
# with the string "UNKNOWN". This is basically never what anyone
# actually wants so we'll just go ahead and delete anything whose
# value is UNKNOWN.
if value.strip() == "UNKNOWN":
del request.POST[key]
# Escape NUL characters, which psycopg doesn't like
if "\x00" in value:
request.POST[key] = value.replace("\x00", "\\x00")
# We require protocol_version 1, it's the only supported version however
# passing a different version should raise an error.
if request.POST.get("protocol_version", "1") != "1":
raise _exc_with_message(HTTPBadRequest, "Unknown protocol version.")
# Check if any fields were supplied as a tuple and have become a
# FieldStorage. The 'content' and 'gpg_signature' fields _should_ be a
# FieldStorage, however.
# ref: https://github.com/pypa/warehouse/issues/2185
# ref: https://github.com/pypa/warehouse/issues/2491
for field in set(request.POST) - {"content", "gpg_signature"}:
values = request.POST.getall(field)
if any(isinstance(value, FieldStorage) for value in values):
raise _exc_with_message(HTTPBadRequest, f"{field}: Should not be a tuple.")
# Look up all of the valid classifiers
all_classifiers = request.db.query(Classifier).all()
# Validate and process the incoming metadata.
form = MetadataForm(request.POST)
# Add a validator for deprecated classifiers
form.classifiers.validators.append(_no_deprecated_classifiers(request))
form.classifiers.choices = [(c.classifier, c.classifier) for c in all_classifiers]
if not form.validate():
for field_name in _error_message_order:
if field_name in form.errors:
break
else:
field_name = sorted(form.errors.keys())[0]
if field_name in form:
field = form[field_name]
if field.description and isinstance(field, wtforms.StringField):
error_message = (
"{value!r} is an invalid value for {field}. ".format(
value=field.data, field=field.description
)
+ "Error: {} ".format(form.errors[field_name][0])
+ "See "
"https://packaging.python.org/specifications/core-metadata"
)
else:
error_message = "Invalid value for {field}. Error: {msgs[0]}".format(
field=field_name, msgs=form.errors[field_name]
)
else:
error_message = "Error: {}".format(form.errors[field_name][0])
raise _exc_with_message(HTTPBadRequest, error_message)
# Ensure that we have file data in the request.
if "content" not in request.POST:
raise _exc_with_message(HTTPBadRequest, "Upload payload does not have a file.")
# Look up the project first before doing anything else, this is so we can
# automatically register it if we need to and can check permissions before
# going any further.
try:
project = (
request.db.query(Project)
.filter(
Project.normalized_name == func.normalize_pep426_name(form.name.data)
)
.one()
)
except NoResultFound:
# Check for AdminFlag set by a PyPI Administrator disabling new project
# registration, reasons for this include Spammers, security
# vulnerabilities, or just wanting to be lazy and not worry ;)
if request.flags.enabled("disallow-new-project-registration"):
raise _exc_with_message(
HTTPForbidden,
(
"New project registration temporarily disabled. "
"See {projecthelp} for details"
).format(projecthelp=request.help_url(_anchor="admin-intervention")),
) from None
# Before we create the project, we're going to check our blacklist to
# see if this project is even allowed to be registered. If it is not,
# then we're going to deny the request to create this project.
if request.db.query(
exists().where(
BlacklistedProject.name == func.normalize_pep426_name(form.name.data)
)
).scalar():
raise _exc_with_message(
HTTPBadRequest,
(
"The name {name!r} isn't allowed. "
"See {projecthelp} "
"for more information."
).format(
name=form.name.data,
projecthelp=request.help_url(_anchor="project-name"),
),
) from None
# Also check for collisions with Python Standard Library modules.
if packaging.utils.canonicalize_name(form.name.data) in STDLIB_PROHIBITTED:
raise _exc_with_message(
HTTPBadRequest,
(
"The name {name!r} isn't allowed (conflict with Python "
"Standard Library module name). See "
"{projecthelp} for more information."
).format(
name=form.name.data,
projecthelp=request.help_url(_anchor="project-name"),
),
) from None
# The project doesn't exist in our database, so first we'll check for
# projects with a similar name
squattees = (
request.db.query(Project)
.filter(
func.levenshtein(
Project.normalized_name, func.normalize_pep426_name(form.name.data)
)
<= 2
)
.all()
)
# Next we'll create the project
project = Project(name=form.name.data)
request.db.add(project)
# Now that the project exists, add any squats which it is the squatter for
for squattee in squattees:
request.db.add(Squat(squatter=project, squattee=squattee))
# Then we'll add a role setting the current user as the "Owner" of the
# project.
request.db.add(Role(user=request.user, project=project, role_name="Owner"))
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=project.name,
action="create",
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
request.db.add(
JournalEntry(
name=project.name,
action="add Owner {}".format(request.user.username),
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
# Check that the user has permission to do things to this project, if this
# is a new project this will act as a sanity check for the role we just
# added above.
if not request.has_permission("upload", project):
raise _exc_with_message(
HTTPForbidden,
(
"The user '{0}' isn't allowed to upload to project '{1}'. "
"See {2} for more information."
).format(
request.user.username,
project.name,
request.help_url(_anchor="project-name"),
),
)
# Uploading should prevent broken rendered descriptions.
# Temporarily disabled, see
# https://github.com/pypa/warehouse/issues/4079
# if form.description.data:
# description_content_type = form.description_content_type.data
# if not description_content_type:
# description_content_type = "text/x-rst"
# rendered = readme.render(
# form.description.data, description_content_type, use_fallback=False
# )
# if rendered is None:
# if form.description_content_type.data:
# message = (
# "The description failed to render "
# "for '{description_content_type}'."
# ).format(description_content_type=description_content_type)
# else:
# message = (
# "The description failed to render "
# "in the default format of reStructuredText."
# )
# raise _exc_with_message(
# HTTPBadRequest,
# "{message} See {projecthelp} for more information.".format(
# message=message,
# projecthelp=request.help_url(_anchor="description-content-type"),
# ),
# ) from None
try:
canonical_version = packaging.utils.canonicalize_version(form.version.data)
release = (
request.db.query(Release)
.filter(
(Release.project == project)
& (Release.canonical_version == canonical_version)
)
.one()
)
except MultipleResultsFound:
# There are multiple releases of this project which have the same
# canonical version that were uploaded before we checked for
# canonical version equivalence, so return the exact match instead
release = (
request.db.query(Release)
.filter(
(Release.project == project) & (Release.version == form.version.data)
)
.one()
)
except NoResultFound:
release = Release(
project=project,
_classifiers=[
c for c in all_classifiers if c.classifier in form.classifiers.data
],
dependencies=list(
_construct_dependencies(
form,
{
"requires": DependencyKind.requires,
"provides": DependencyKind.provides,
"obsoletes": DependencyKind.obsoletes,
"requires_dist": DependencyKind.requires_dist,
"provides_dist": DependencyKind.provides_dist,
"obsoletes_dist": DependencyKind.obsoletes_dist,
"requires_external": DependencyKind.requires_external,
"project_urls": DependencyKind.project_url,
},
)
),
canonical_version=canonical_version,
**{
k: getattr(form, k).data
for k in {
# This is a list of all the fields in the form that we
# should pull off and insert into our new release.
"version",
"summary",
"description",
"description_content_type",
"license",
"author",
"author_email",
"maintainer",
"maintainer_email",
"keywords",
"platform",
"home_page",
"download_url",
"requires_python",
}
},
uploader=request.user,
uploaded_via=request.user_agent,
)
request.db.add(release)
# TODO: This should be handled by some sort of database trigger or
# a SQLAlchemy hook or the like instead of doing it inline in
# this view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="new release",
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
# TODO: We need a better solution to this than to just do it inline inside
# this method. Ideally the version field would just be sortable, but
# at least this should be some sort of hook or trigger.
releases = (
request.db.query(Release)
.filter(Release.project == project)
.options(orm.load_only(Release._pypi_ordering))
.all()
)
for i, r in enumerate(
sorted(releases, key=lambda x: packaging.version.parse(x.version))
):
r._pypi_ordering = i
# Pull the filename out of our POST data.
filename = request.POST["content"].filename
# Make sure that the filename does not contain any path separators.
if "/" in filename or "\\" in filename:
raise _exc_with_message(
HTTPBadRequest, "Cannot upload a file with '/' or '\\' in the name."
)
# Make sure the filename ends with an allowed extension.
if _dist_file_regexes[project.allow_legacy_files].search(filename) is None:
raise _exc_with_message(
HTTPBadRequest,
"Invalid file extension: Use .egg, .tar.gz, .whl or .zip "
"extension. (https://www.python.org/dev/peps/pep-0527)",
)
# Make sure that our filename matches the project that it is being uploaded
# to.
prefix = pkg_resources.safe_name(project.name).lower()
if not pkg_resources.safe_name(filename).lower().startswith(prefix):
raise _exc_with_message(
HTTPBadRequest,
"Start filename for {!r} with {!r}.".format(project.name, prefix),
)
# Check the content type of what is being uploaded
if not request.POST["content"].type or request.POST["content"].type.startswith(
"image/"
):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Ensure that the package filetype is allowed.
# TODO: Once PEP 527 is completely implemented we should be able to delete
# this and just move it into the form itself.
if not project.allow_legacy_files and form.filetype.data not in {
"sdist",
"bdist_wheel",
"bdist_egg",
}:
raise _exc_with_message(HTTPBadRequest, "Unknown type of file.")
# The project may or may not have a file size specified on the project, if
# it does then it may or may not be smaller or larger than our global file
# size limits.
file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
with tempfile.TemporaryDirectory() as tmpdir:
temporary_filename = os.path.join(tmpdir, filename)
# Buffer the entire file onto disk, checking the hash of the file as we
# go along.
with open(temporary_filename, "wb") as fp:
file_size = 0
file_hashes = {
"md5": hashlib.md5(),
"sha256": hashlib.sha256(),
"blake2_256": hashlib.blake2b(digest_size=256 // 8),
}
for chunk in iter(lambda: request.POST["content"].file.read(8096), b""):
file_size += len(chunk)
if file_size > file_size_limit:
raise _exc_with_message(
HTTPBadRequest,
"File too large. "
+ "Limit for project {name!r} is {limit} MB. ".format(
name=project.name, limit=file_size_limit // (1024 * 1024)
)
+ "See "
+ request.help_url(_anchor="file-size-limit"),
)
fp.write(chunk)
for hasher in file_hashes.values():
hasher.update(chunk)
# Take our hash functions and compute the final hashes for them now.
file_hashes = {k: h.hexdigest().lower() for k, h in file_hashes.items()}
# Actually verify the digests that we've gotten. We're going to use
# hmac.compare_digest even though we probably don't actually need to
# because it's better safe than sorry. In the case of multiple digests
# we expect them all to be given.
if not all(
[
hmac.compare_digest(
getattr(form, "{}_digest".format(digest_name)).data.lower(),
digest_value,
)
for digest_name, digest_value in file_hashes.items()
if getattr(form, "{}_digest".format(digest_name)).data
]
):
raise _exc_with_message(
HTTPBadRequest,
"The digest supplied does not match a digest calculated "
"from the uploaded file.",
)
# Check to see if the file that was uploaded exists already or not.
is_duplicate = _is_duplicate_file(request.db, filename, file_hashes)
if is_duplicate:
return Response()
elif is_duplicate is not None:
raise _exc_with_message(
HTTPBadRequest,
# Note: Changing this error message to something that doesn't
# start with "File already exists" will break the
# --skip-existing functionality in twine
# ref: https://github.com/pypa/warehouse/issues/3482
# ref: https://github.com/pypa/twine/issues/332
"File already exists. See "
+ request.help_url(_anchor="file-name-reuse"),
)
# Check to see if the file that was uploaded exists in our filename log
if request.db.query(
request.db.query(Filename).filter(Filename.filename == filename).exists()
).scalar():
raise _exc_with_message(
HTTPBadRequest,
"This filename has already been used, use a "
"different version. "
"See " + request.help_url(_anchor="file-name-reuse"),
)
# Check to see if uploading this file would create a duplicate sdist
# for the current release.
if (
form.filetype.data == "sdist"
and request.db.query(
request.db.query(File)
.filter((File.release == release) & (File.packagetype == "sdist"))
.exists()
).scalar()
):
raise _exc_with_message(
HTTPBadRequest, "Only one sdist may be uploaded per release."
)
# Check the file to make sure it is a valid distribution file.
if not _is_valid_dist_file(temporary_filename, form.filetype.data):
raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
# Check that if it's a binary wheel, it's on a supported platform
if filename.endswith(".whl"):
wheel_info = _wheel_file_re.match(filename)
plats = wheel_info.group("plat").split(".")
for plat in plats:
if not _valid_platform_tag(plat):
raise _exc_with_message(
HTTPBadRequest,
"Binary wheel '{filename}' has an unsupported "
"platform tag '{plat}'.".format(filename=filename, plat=plat),
)
# Also buffer the entire signature file to disk.
if "gpg_signature" in request.POST:
has_signature = True
with open(os.path.join(tmpdir, filename + ".asc"), "wb") as fp:
signature_size = 0
for chunk in iter(
lambda: request.POST["gpg_signature"].file.read(8096), b""
):
signature_size += len(chunk)
if signature_size > MAX_SIGSIZE:
raise _exc_with_message(HTTPBadRequest, "Signature too large.")
fp.write(chunk)
# Check whether signature is ASCII armored
with open(os.path.join(tmpdir, filename + ".asc"), "rb") as fp:
if not fp.read().startswith(b"-----BEGIN PGP SIGNATURE-----"):
raise _exc_with_message(
HTTPBadRequest, "PGP signature isn't ASCII armored."
)
else:
has_signature = False
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(Filename(filename=filename))
# Store the information about the file in the database.
file_ = File(
release=release,
filename=filename,
python_version=form.pyversion.data,
packagetype=form.filetype.data,
comment_text=form.comment.data,
size=file_size,
has_signature=bool(has_signature),
md5_digest=file_hashes["md5"],
sha256_digest=file_hashes["sha256"],
blake2_256_digest=file_hashes["blake2_256"],
# Figure out what our filepath is going to be, we're going to use a
# directory structure based on the hash of the file contents. This
# will ensure that the contents of the file cannot change without
# it also changing the path that the file is saved too.
path="/".join(
[
file_hashes[PATH_HASHER][:2],
file_hashes[PATH_HASHER][2:4],
file_hashes[PATH_HASHER][4:],
filename,
]
),
uploaded_via=request.user_agent,
)
request.db.add(file_)
# TODO: This should be handled by some sort of database trigger or a
# SQLAlchemy hook or the like instead of doing it inline in this
# view.
request.db.add(
JournalEntry(
name=release.project.name,
version=release.version,
action="add {python_version} file {filename}".format(
python_version=file_.python_version, filename=file_.filename
),
submitted_by=request.user,
submitted_from=request.remote_addr,
)
)
# TODO: We need a better answer about how to make this transactional so
# this won't take affect until after a commit has happened, for
# now we'll just ignore it and save it before the transaction is
# committed.
storage = request.find_service(IFileStorage)
storage.store(
file_.path,
os.path.join(tmpdir, filename),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
if has_signature:
storage.store(
file_.pgp_path,
os.path.join(tmpdir, filename + ".asc"),
meta={
"project": file_.release.project.normalized_name,
"version": file_.release.version,
"package-type": file_.packagetype,
"python-version": file_.python_version,
},
)
return Response()