def _project_docs(db, project_name=None):
releases_list = (db.query(Release.name, Release.version).order_by(
Release.name,
Release.is_prerelease.nullslast(),
Release._pypi_ordering.desc(),
).distinct(Release.name))
if project_name:
releases_list = releases_list.filter(Release.name == project_name)
releases_list = releases_list.subquery()
r = aliased(Release, name="r")
all_versions = (db.query(func.array_agg(
r.version)).filter(r.name == Release.name).correlate(
Release).as_scalar().label("all_versions"))
classifiers = (db.query(func.array_agg(
Classifier.classifier)).select_from(release_classifiers).join(
Classifier,
Classifier.id == release_classifiers.c.trove_id).filter(
Release.name == release_classifiers.c.name).filter(
Release.version == release_classifiers.c.version).
correlate(Release).as_scalar().label("classifiers"))
release_data = (db.query(
Release.description,
Release.name,
Release.version.label("latest_version"),
all_versions,
Release.author,
Release.author_email,
Release.maintainer,
Release.maintainer_email,
Release.home_page,
Release.summary,
Release.keywords,
Release.platform,
Release.download_url,
Release.created,
classifiers,
Project.normalized_name,
Project.name,
).select_from(releases_list).join(
Release,
and_(
Release.name == releases_list.c.name,
Release.version == releases_list.c.version,
),
).outerjoin(Release.project).order_by(Release.name))
for release in windowed_query(release_data, Release.name, 50000):
p = ProjectDocument.from_db(release)
p._index = None
p.full_clean()
doc = p.to_dict(include_meta=True)
doc.pop("_index", None)
yield doc
def _project_docs(db):
releases = (db.query(Release).execution_options(
stream_results=True).options(
lazyload("*"),
joinedload(Release.project).subqueryload(
Project.releases).load_only("version")).distinct(
Release.name).order_by(Release.name,
Release._pypi_ordering.desc()))
for release in releases:
p = ProjectDocType.from_db(release)
p.full_clean()
yield p.to_dict(include_meta=True)
def _project_docs(db):
releases = (
db.query(Release)
.execution_options(stream_results=True)
.options(lazyload("*"),
joinedload(Release.project)
.subqueryload(Project.releases)
.load_only("version"))
.distinct(Release.name)
.order_by(Release.name, Release._pypi_ordering.desc())
)
for release in releases:
p = ProjectDocType.from_db(release)
p.full_clean()
yield p.to_dict(include_meta=True)
def _project_docs(db):
releases = (db.query(Release).options(
load_only(
"summary", "description", "author", "author_email", "maintainer",
"maintainer_email", "home_page", "download_url", "keywords",
"platform", "created")).options(
lazyload("*"), (joinedload(Release.project).load_only(
"normalized_name", "name").joinedload(
Project.releases).load_only("version")),
joinedload(
Release._classifiers).load_only("classifier")).distinct(
Release.name).order_by(Release.name,
Release._pypi_ordering.desc()))
for release in windowed_query(releases, Release.name, 1000):
p = ProjectDocType.from_db(release)
p.full_clean()
yield p.to_dict(include_meta=True)
def test_build_search():
release = pretend.stub(
project=pretend.stub(
name="Foobar",
normalized_name="foobar",
releases=[
pretend.stub(version="1.0", is_prerelease=False),
pretend.stub(version="2.0", is_prerelease=False),
pretend.stub(version="3.0", is_prerelease=False),
pretend.stub(version="4.0", is_prerelease=False),
pretend.stub(version="5.0.dev0", is_prerelease=True),
],
),
summary="This is my summary",
description="This is my description",
author="Jane Author",
author_email="*****@*****.**",
maintainer="Joe Maintainer",
maintainer_email="*****@*****.**",
home_page="https://example.com/foobar/",
download_url="https://example.com/foobar/downloads/",
keywords="the, keywords, lol",
platform="any platform",
created=datetime.datetime(1956, 1, 31),
classifiers=["Alpha", "Beta"],
)
obj = Project.from_db(release)
assert obj.meta.id == "foobar"
assert obj["name"] == "Foobar"
assert obj["version"] == ["5.0.dev0", "4.0", "3.0", "2.0", "1.0"]
assert obj["latest_version"] == "4.0"
assert obj["summary"] == "This is my summary"
assert obj["description"] == "This is my description"
assert obj["author"] == "Jane Author"
assert obj["author_email"] == "*****@*****.**"
assert obj["maintainer"] == "Joe Maintainer"
assert obj["maintainer_email"] == "*****@*****.**"
assert obj["home_page"] == "https://example.com/foobar/"
assert obj["download_url"] == "https://example.com/foobar/downloads/"
assert obj["keywords"] == "the, keywords, lol"
assert obj["platform"] == "any platform"
assert obj["created"] == datetime.datetime(1956, 1, 31)
assert obj["classifiers"] == ['Alpha', 'Beta']
def test_build_search():
release = pretend.stub(
name="Foobar",
normalized_name="foobar",
all_versions=[
"5.0.dev0",
"4.0",
"3.0",
"2.0",
"1.0",
],
latest_version="4.0",
summary="This is my summary",
description="This is my description",
author="Jane Author",
author_email="*****@*****.**",
maintainer="Joe Maintainer",
maintainer_email="*****@*****.**",
home_page="https://example.com/foobar/",
download_url="https://example.com/foobar/downloads/",
keywords="the, keywords, lol",
platform="any platform",
created=datetime.datetime(1956, 1, 31),
classifiers=["Alpha", "Beta"],
)
obj = Project.from_db(release)
assert obj.meta.id == "foobar"
assert obj["name"] == "Foobar"
assert obj["version"] == ["5.0.dev0", "4.0", "3.0", "2.0", "1.0"]
assert obj["latest_version"] == "4.0"
assert obj["summary"] == "This is my summary"
assert obj["description"] == "This is my description"
assert obj["author"] == "Jane Author"
assert obj["author_email"] == "*****@*****.**"
assert obj["maintainer"] == "Joe Maintainer"
assert obj["maintainer_email"] == "*****@*****.**"
assert obj["home_page"] == "https://example.com/foobar/"
assert obj["download_url"] == "https://example.com/foobar/downloads/"
assert obj["keywords"] == "the, keywords, lol"
assert obj["platform"] == "any platform"
assert obj["created"] == datetime.datetime(1956, 1, 31)
assert obj["classifiers"] == ['Alpha', 'Beta']
def _project_docs(db):
releases = (
db.query(Release)
.options(load_only(
"summary", "description", "author",
"author_email", "maintainer", "maintainer_email",
"home_page", "download_url", "keywords", "platform",
"created"))
.options(lazyload("*"),
(joinedload(Release.project)
.load_only("normalized_name", "name")
.joinedload(Project.releases)
.load_only("version", "is_prerelease")),
joinedload(Release._classifiers).load_only("classifier"))
.distinct(Release.name)
.order_by(Release.name, Release._pypi_ordering.desc())
)
for release in windowed_query(releases, Release.name, 1000):
p = ProjectDocType.from_db(release)
p.full_clean()
yield p.to_dict(include_meta=True)
def _project_docs(db, project_name=None):
releases_list = (
db.query(Release.id)
.order_by(
Release.project_id,
Release.is_prerelease.nullslast(),
Release._pypi_ordering.desc(),
)
.distinct(Release.project_id)
)
if project_name:
releases_list = releases_list.join(Project).filter(Project.name == project_name)
releases_list = releases_list.subquery()
r = aliased(Release, name="r")
all_versions = (
db.query(func.array_agg(r.version))
.filter(r.project_id == Release.project_id)
.correlate(Release)
.as_scalar()
.label("all_versions")
)
classifiers = (
db.query(func.array_agg(Classifier.classifier))
.select_from(release_classifiers)
.join(Classifier, Classifier.id == release_classifiers.c.trove_id)
.filter(Release.id == release_classifiers.c.release_id)
.correlate(Release)
.as_scalar()
.label("classifiers")
)
release_data = (
db.query(
Release.description,
Release.version.label("latest_version"),
all_versions,
Release.author,
Release.author_email,
Release.maintainer,
Release.maintainer_email,
Release.home_page,
Release.summary,
Release.keywords,
Release.platform,
Release.download_url,
Release.created,
classifiers,
Project.normalized_name,
Project.name,
)
.select_from(releases_list)
.join(Release, Release.id == releases_list.c.id)
.outerjoin(Release.project)
)
for release in windowed_query(release_data, Release.project_id, 50000):
p = ProjectDocument.from_db(release)
p._index = None
p.full_clean()
doc = p.to_dict(include_meta=True)
doc.pop("_index", None)
yield doc