def _project_docs(db, project_name=None): releases_list = (db.query(Release.name, Release.version).order_by( Release.name, Release.is_prerelease.nullslast(), Release._pypi_ordering.desc(), ).distinct(Release.name)) if project_name: releases_list = releases_list.filter(Release.name == project_name) releases_list = releases_list.subquery() r = aliased(Release, name="r") all_versions = (db.query(func.array_agg( r.version)).filter(r.name == Release.name).correlate( Release).as_scalar().label("all_versions")) classifiers = (db.query(func.array_agg( Classifier.classifier)).select_from(release_classifiers).join( Classifier, Classifier.id == release_classifiers.c.trove_id).filter( Release.name == release_classifiers.c.name).filter( Release.version == release_classifiers.c.version). correlate(Release).as_scalar().label("classifiers")) release_data = (db.query( Release.description, Release.name, Release.version.label("latest_version"), all_versions, Release.author, Release.author_email, Release.maintainer, Release.maintainer_email, Release.home_page, Release.summary, Release.keywords, Release.platform, Release.download_url, Release.created, classifiers, Project.normalized_name, Project.name, ).select_from(releases_list).join( Release, and_( Release.name == releases_list.c.name, Release.version == releases_list.c.version, ), ).outerjoin(Release.project).order_by(Release.name)) for release in windowed_query(release_data, Release.name, 50000): p = ProjectDocument.from_db(release) p._index = None p.full_clean() doc = p.to_dict(include_meta=True) doc.pop("_index", None) yield doc
def _project_docs(db): releases = (db.query(Release).execution_options( stream_results=True).options( lazyload("*"), joinedload(Release.project).subqueryload( Project.releases).load_only("version")).distinct( Release.name).order_by(Release.name, Release._pypi_ordering.desc())) for release in releases: p = ProjectDocType.from_db(release) p.full_clean() yield p.to_dict(include_meta=True)
def _project_docs(db): releases = ( db.query(Release) .execution_options(stream_results=True) .options(lazyload("*"), joinedload(Release.project) .subqueryload(Project.releases) .load_only("version")) .distinct(Release.name) .order_by(Release.name, Release._pypi_ordering.desc()) ) for release in releases: p = ProjectDocType.from_db(release) p.full_clean() yield p.to_dict(include_meta=True)
def _project_docs(db): releases = (db.query(Release).options( load_only( "summary", "description", "author", "author_email", "maintainer", "maintainer_email", "home_page", "download_url", "keywords", "platform", "created")).options( lazyload("*"), (joinedload(Release.project).load_only( "normalized_name", "name").joinedload( Project.releases).load_only("version")), joinedload( Release._classifiers).load_only("classifier")).distinct( Release.name).order_by(Release.name, Release._pypi_ordering.desc())) for release in windowed_query(releases, Release.name, 1000): p = ProjectDocType.from_db(release) p.full_clean() yield p.to_dict(include_meta=True)
def test_build_search(): release = pretend.stub( project=pretend.stub( name="Foobar", normalized_name="foobar", releases=[ pretend.stub(version="1.0", is_prerelease=False), pretend.stub(version="2.0", is_prerelease=False), pretend.stub(version="3.0", is_prerelease=False), pretend.stub(version="4.0", is_prerelease=False), pretend.stub(version="5.0.dev0", is_prerelease=True), ], ), summary="This is my summary", description="This is my description", author="Jane Author", author_email="*****@*****.**", maintainer="Joe Maintainer", maintainer_email="*****@*****.**", home_page="https://example.com/foobar/", download_url="https://example.com/foobar/downloads/", keywords="the, keywords, lol", platform="any platform", created=datetime.datetime(1956, 1, 31), classifiers=["Alpha", "Beta"], ) obj = Project.from_db(release) assert obj.meta.id == "foobar" assert obj["name"] == "Foobar" assert obj["version"] == ["5.0.dev0", "4.0", "3.0", "2.0", "1.0"] assert obj["latest_version"] == "4.0" assert obj["summary"] == "This is my summary" assert obj["description"] == "This is my description" assert obj["author"] == "Jane Author" assert obj["author_email"] == "*****@*****.**" assert obj["maintainer"] == "Joe Maintainer" assert obj["maintainer_email"] == "*****@*****.**" assert obj["home_page"] == "https://example.com/foobar/" assert obj["download_url"] == "https://example.com/foobar/downloads/" assert obj["keywords"] == "the, keywords, lol" assert obj["platform"] == "any platform" assert obj["created"] == datetime.datetime(1956, 1, 31) assert obj["classifiers"] == ['Alpha', 'Beta']
def test_build_search(): release = pretend.stub( name="Foobar", normalized_name="foobar", all_versions=[ "5.0.dev0", "4.0", "3.0", "2.0", "1.0", ], latest_version="4.0", summary="This is my summary", description="This is my description", author="Jane Author", author_email="*****@*****.**", maintainer="Joe Maintainer", maintainer_email="*****@*****.**", home_page="https://example.com/foobar/", download_url="https://example.com/foobar/downloads/", keywords="the, keywords, lol", platform="any platform", created=datetime.datetime(1956, 1, 31), classifiers=["Alpha", "Beta"], ) obj = Project.from_db(release) assert obj.meta.id == "foobar" assert obj["name"] == "Foobar" assert obj["version"] == ["5.0.dev0", "4.0", "3.0", "2.0", "1.0"] assert obj["latest_version"] == "4.0" assert obj["summary"] == "This is my summary" assert obj["description"] == "This is my description" assert obj["author"] == "Jane Author" assert obj["author_email"] == "*****@*****.**" assert obj["maintainer"] == "Joe Maintainer" assert obj["maintainer_email"] == "*****@*****.**" assert obj["home_page"] == "https://example.com/foobar/" assert obj["download_url"] == "https://example.com/foobar/downloads/" assert obj["keywords"] == "the, keywords, lol" assert obj["platform"] == "any platform" assert obj["created"] == datetime.datetime(1956, 1, 31) assert obj["classifiers"] == ['Alpha', 'Beta']
def _project_docs(db): releases = ( db.query(Release) .options(load_only( "summary", "description", "author", "author_email", "maintainer", "maintainer_email", "home_page", "download_url", "keywords", "platform", "created")) .options(lazyload("*"), (joinedload(Release.project) .load_only("normalized_name", "name") .joinedload(Project.releases) .load_only("version", "is_prerelease")), joinedload(Release._classifiers).load_only("classifier")) .distinct(Release.name) .order_by(Release.name, Release._pypi_ordering.desc()) ) for release in windowed_query(releases, Release.name, 1000): p = ProjectDocType.from_db(release) p.full_clean() yield p.to_dict(include_meta=True)
def _project_docs(db, project_name=None): releases_list = ( db.query(Release.id) .order_by( Release.project_id, Release.is_prerelease.nullslast(), Release._pypi_ordering.desc(), ) .distinct(Release.project_id) ) if project_name: releases_list = releases_list.join(Project).filter(Project.name == project_name) releases_list = releases_list.subquery() r = aliased(Release, name="r") all_versions = ( db.query(func.array_agg(r.version)) .filter(r.project_id == Release.project_id) .correlate(Release) .as_scalar() .label("all_versions") ) classifiers = ( db.query(func.array_agg(Classifier.classifier)) .select_from(release_classifiers) .join(Classifier, Classifier.id == release_classifiers.c.trove_id) .filter(Release.id == release_classifiers.c.release_id) .correlate(Release) .as_scalar() .label("classifiers") ) release_data = ( db.query( Release.description, Release.version.label("latest_version"), all_versions, Release.author, Release.author_email, Release.maintainer, Release.maintainer_email, Release.home_page, Release.summary, Release.keywords, Release.platform, Release.download_url, Release.created, classifiers, Project.normalized_name, Project.name, ) .select_from(releases_list) .join(Release, Release.id == releases_list.c.id) .outerjoin(Release.project) ) for release in windowed_query(release_data, Release.project_id, 50000): p = ProjectDocument.from_db(release) p._index = None p.full_clean() doc = p.to_dict(include_meta=True) doc.pop("_index", None) yield doc