def blacklist(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None blacklist_query = request.db.query(BlacklistedProject).order_by( BlacklistedProject.name ) if q: terms = shlex.split(q) filters = [] for term in terms: filters.append( BlacklistedProject.name.ilike(func.normalize_pep426_name(term)) ) blacklist_query = blacklist_query.filter(or_(*filters)) blacklist = SQLAlchemyORMPage( blacklist_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"blacklist": blacklist, "query": q}
def test_manually_reviewed_filter(self, db_request, manually_reviewed): check = MalwareCheckFactory.create() for _ in range(5): MalwareVerdictFactory.create( check=check, manually_reviewed=bool(manually_reviewed)) # Create other verdicts to ensure filter works properly for _ in range(10): MalwareVerdictFactory.create( check=check, manually_reviewed=not bool(manually_reviewed)) db_request.GET["manually_reviewed"] = str(manually_reviewed) query = (db_request.db.query(MalwareVerdict).filter( MalwareVerdict.manually_reviewed == bool( manually_reviewed)).order_by(MalwareVerdict.run_date.desc())) verdicts = SQLAlchemyORMPage( query, page=1, items_per_page=25, url_maker=paginate_url_factory(db_request), ) response = { "verdicts": verdicts, "check_names": set([check.name]), "classifications": set(["threat", "indeterminate", "benign"]), "confidences": set(["low", "medium", "high"]), } assert views.get_verdicts(db_request) == response
def search(request): if request.params.get("q"): query = request.es.query( "multi_match", query=request.params["q"], fields=[ "name", "version", "author", "author_email", "maintainer", "maintainer_email", "home_page", "license", "summary", "description", "keywords", "platform", "download_url", ], ).suggest( name="name_suggestion", text=request.params["q"], term={"field": "name"} ) else: query = request.es.query() page = ElasticsearchPage( query, page=int(request.params.get("page", 1)), url_maker=paginate_url_factory(request), ) return {"page": page, "term": request.params.get("q")}
def project_list(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None projects_query = request.db.query(Project).order_by(Project.name) if q: terms = shlex.split(q) filters = [] for term in terms: filters.append(Project.name.ilike(term)) projects_query = projects_query.filter(or_(*filters)) projects = SQLAlchemyORMPage( projects_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"projects": projects, "query": q}
def email_list(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None email_query = request.db.query(EmailMessage).order_by( EmailMessage.created.desc(), EmailMessage.id) if q: terms = shlex.split(q) filters = [] for term in terms: filters.append(EmailMessage.to.ilike(term)) email_query = email_query.filter(or_(*filters)) emails = SQLAlchemyORMPage( email_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"emails": emails, "query": q}
def search(request): if request.params.get("q"): query = request.es.query( "multi_match", query=request.params["q"], fields=[ "name", "version", "author", "author_email", "maintainer", "maintainer_email", "home_page", "license", "summary", "description", "keywords", "platform", "download_url", ], ) else: query = request.es.query() page = ElasticsearchPage( query, page=int(request.params.get("page", 1)), url_maker=paginate_url_factory(request), ) return {"page": page}
def search(request): q = request.params.get("q", '') if q: should = [] for field in SEARCH_FIELDS: kw = {"query": q} if field in SEARCH_BOOSTS: kw["boost"] = SEARCH_BOOSTS[field] should.append(Q("match", **{field: kw})) # Add a prefix query if ``q`` is longer than one character. if len(q) > 1: should.append(Q('prefix', normalized_name=q)) query = request.es.query("dis_max", queries=should) query = query.suggest("name_suggestion", q, term={"field": "name"}) else: query = request.es.query() if request.params.get("o"): query = query.sort(request.params["o"]) if request.params.getall("c"): query = query.filter("terms", classifiers=request.params.getall("c")) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") page = ElasticsearchPage( query, page=page_num, url_maker=paginate_url_factory(request), ) if page.page_count and page_num > page.page_count: return HTTPNotFound() available_filters = collections.defaultdict(list) for cls in request.db.query(Classifier).order_by(Classifier.classifier): first, *_ = cls.classifier.split(' :: ') available_filters[first].append(cls.classifier) def filter_key(item): try: return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0] except ValueError: return 1, 0, item[0] return { "page": page, "term": q, "order": request.params.get("o", ''), "available_filters": sorted(available_filters.items(), key=filter_key), "applied_filters": request.params.getall("c"), }
def test_classification_filter(self, db_request, classification): check = MalwareCheckFactory.create() for c in VerdictClassification: for _ in range(5): MalwareVerdictFactory.create(check=check, classification=c) db_request.GET["classification"] = classification query = db_request.db.query(MalwareVerdict) if classification: query = query.filter(MalwareVerdict.classification == classification) query = query.order_by(MalwareVerdict.run_date.desc()) verdicts = SQLAlchemyORMPage( query, page=1, items_per_page=25, url_maker=paginate_url_factory(db_request), ) response = { "verdicts": verdicts, "check_names": set([check.name]), "classifications": set(["threat", "indeterminate", "benign"]), "confidences": set(["low", "medium", "high"]), } assert views.get_verdicts(db_request) == response
def test_check_name_filter(self, db_request, check_name): for i in range(3): check = MalwareCheckFactory.create(name="check%d" % i) for _ in range(5): MalwareVerdictFactory.create(check=check) query = db_request.db.query(MalwareVerdict) if check_name: query = query.join(MalwareCheck).filter(MalwareCheck.name == check_name) query = query.order_by(MalwareVerdict.run_date.desc()) verdicts = SQLAlchemyORMPage( query, page=1, items_per_page=25, url_maker=paginate_url_factory(db_request), ) response = { "verdicts": verdicts, "check_names": set(["check0", "check1", "check2"]), "classifications": set(["threat", "indeterminate", "benign"]), "confidences": set(["low", "medium", "high"]), } db_request.GET["check_name"] = check_name assert views.get_verdicts(db_request) == response
def user_list(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None users_query = request.db.query(User).order_by(User.username) if q: terms = shlex.split(q) filters = [] for term in terms: if ":" in term: field, value = term.split(":", 1) if field.lower() == "email": filters.append(User.emails.any(Email.email.ilike(value))) else: filters.append(User.username.ilike(term)) users_query = users_query.filter(or_(*filters)) users = SQLAlchemyORMPage( users_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"users": users, "query": q}
def project_list(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None projects_query = request.db.query(Project).order_by(Project.normalized_name) if q: terms = shlex.split(q) filters = [] for term in terms: filters.append(Project.name.ilike(term)) projects_query = projects_query.filter(or_(*filters)) projects = SQLAlchemyORMPage( projects_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"projects": projects, "query": q}
def project_list(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None projects_query = request.db.query(Project).order_by( Project.normalized_name) exact_match = None if q: projects_query = projects_query.filter( func.ultranormalize_name(Project.name) == func.ultranormalize_name( q)) exact_match = (request.db.query(Project).filter( Project.normalized_name == q).one_or_none()) projects = SQLAlchemyORMPage( projects_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"projects": projects, "query": q, "exact_match": exact_match}
def email_list(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None email_query = request.db.query(EmailMessage).order_by( EmailMessage.created.desc(), EmailMessage.id ) if q: terms = shlex.split(q) filters = [] for term in terms: filters.append(EmailMessage.to.ilike(term)) email_query = email_query.filter(or_(*filters)) emails = SQLAlchemyORMPage( email_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"emails": emails, "query": q}
def search(request): q = request.params.get("q", '') if q: should = [] for field in SEARCH_FIELDS: kw = {"query": q} if field in SEARCH_BOOSTS: kw["boost"] = SEARCH_BOOSTS[field] should.append(Q("match", **{field: kw})) # Add a prefix query if ``q`` is longer than one character. if len(q) > 1: should.append(Q('prefix', normalized_name=q)) query = request.es.query("dis_max", queries=should) query = query.suggest("name_suggestion", q, term={"field": "name"}) else: query = request.es.query() if request.params.get("o"): query = query.sort(request.params["o"]) if request.params.getall("c"): query = query.filter("terms", classifiers=request.params.getall("c")) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") page = ElasticsearchPage( query, page=page_num, url_maker=paginate_url_factory(request), ) if page.page_count and page_num > page.page_count: raise HTTPNotFound available_filters = collections.defaultdict(list) for cls in request.db.query(Classifier).order_by(Classifier.classifier): first, *_ = cls.classifier.split(' :: ') available_filters[first].append(cls.classifier) return { "page": page, "term": q, "order": request.params.get("o", ''), "available_filters": sorted(available_filters.items()), "applied_filters": request.params.getall("c"), }
def search(request): if request.params.get("q"): query = request.es.query( "multi_match", query=request.params["q"], fields=[ "author", "author_email", "description^5", "download_url", "home_page", "keywords^5", "license", "maintainer", "maintainer_email", "normalized_name^10", "platform", "summary^5", ], ).suggest(name="name_suggestion", text=request.params["q"], term={"field": "name"}) else: query = request.es.query() if request.params.get("o"): query = query.sort(request.params["o"]) if request.params.getall("c"): query = query.filter('terms', classifiers=request.params.getall("c")) page_num = int(request.params.get("page", 1)) page = ElasticsearchPage( query, page=page_num, url_maker=paginate_url_factory(request), ) if page.page_count and page_num > page.page_count: raise HTTPNotFound available_filters = collections.defaultdict(list) for cls in request.db.query(Classifier).order_by(Classifier.classifier): first, *_ = cls.classifier.split(' :: ') available_filters[first].append(cls.classifier) return { "page": page, "term": request.params.get("q", ''), "order": request.params.get("o", ''), "available_filters": sorted(available_filters.items()), "applied_filters": request.params.getall("c"), }
def test_paginate_url(pyramid_request): pyramid_request.GET = MultiDict(pyramid_request.GET) pyramid_request.GET["foo"] = "bar" url = pretend.stub() pyramid_request.current_route_path = pretend.call_recorder(lambda _query: url) url_maker = paginate.paginate_url_factory(pyramid_request) assert url_maker(5) is url assert pyramid_request.current_route_path.calls == [ pretend.call(_query=[("foo", "bar"), ("page", 5)]) ]
def test_paginate_url(pyramid_request): pyramid_request.GET = MultiDict(pyramid_request.GET) pyramid_request.GET["foo"] = "bar" url = pretend.stub() pyramid_request.current_route_path = \ pretend.call_recorder(lambda _query: url) url_maker = paginate.paginate_url_factory(pyramid_request) assert url_maker(5) is url assert pyramid_request.current_route_path.calls == [ pretend.call(_query=[("foo", "bar"), ("page", 5)]), ]
def search(request): if request.params.get("q"): query = request.es.query( "multi_match", query=request.params["q"], fields=[ "author", "author_email", "description^5", "download_url", "home_page", "keywords^5", "license", "maintainer", "maintainer_email", "normalized_name^10", "platform", "summary^5", ], ).suggest( name="name_suggestion", text=request.params["q"], term={"field": "name"} ) else: query = request.es.query() if request.params.get("o"): query = query.sort(request.params["o"]) if request.params.getall("c"): query = query.filter('terms', classifiers=request.params.getall("c")) page_num = int(request.params.get("page", 1)) page = ElasticsearchPage( query, page=page_num, url_maker=paginate_url_factory(request), ) if page.page_count and page_num > page.page_count: raise HTTPNotFound available_filters = collections.defaultdict(list) for cls in request.db.query(Classifier).order_by(Classifier.classifier): first, *_ = cls.classifier.split(' :: ') available_filters[first].append(cls.classifier) return { "page": page, "term": request.params.get("q", ''), "order": request.params.get("o", ''), "available_filters": sorted(available_filters.items()), "applied_filters": request.params.getall("c"), }
def get_verdicts(request): result = {} result["check_names"] = set( [name for (name, ) in request.db.query(MalwareCheck.name)]) result["classifications"] = set([c.value for c in VerdictClassification]) result["confidences"] = set([c.value for c in VerdictConfidence]) validate_fields(request, result) result["verdicts"] = SQLAlchemyORMPage( generate_query(request.db, request.params), page=int(request.params.get("page", 1)), items_per_page=25, url_maker=paginate_url_factory(request), ) return result
def releases_list(project, request): q = request.params.get("q") project_name = request.matchdict["project_name"] if project_name != project.normalized_name: raise HTTPMovedPermanently( request.current_route_path( project_name=project.normalized_name, ), ) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None releases_query = (request.db.query(Release) .filter(Release.project == project) .order_by(Release._pypi_ordering.desc())) if q: terms = shlex.split(q) filters = [] for term in terms: if ":" in term: field, value = term.split(":", 1) if field.lower() == "version": filters.append(Release.version.ilike(value)) releases_query = releases_query.filter(or_(*filters)) releases = SQLAlchemyORMPage( releases_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return { "releases": releases, "project": project, "query": q, }
def journals_list(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None journals_query = ( request.db.query(JournalEntry) .order_by( JournalEntry.submitted_date.desc(), JournalEntry.id.desc()) ) if q: terms = shlex.split(q) filters = [] for term in terms: if ":" in term: field, value = term.split(":", 1) if field.lower() == "project": filters.append(JournalEntry.name.ilike(value)) if field.lower() == "version": filters.append(JournalEntry.version.ilike(value)) if field.lower() == "user": filters.append(JournalEntry._submitted_by.like(value)) if field.lower() == "ip": filters.append(JournalEntry.submitted_from.ilike(value)) else: filters.append(JournalEntry.name.ilike(term)) journals_query = journals_query.filter(and_(*filters)) journals = SQLAlchemyORMPage( journals_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"journals": journals, "query": q}
def journals_list(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None journals_query = ( request.db.query(JournalEntry) .options(joinedload(JournalEntry.submitted_by)) .order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc()) ) if q: terms = shlex.split(q) filters = [] for term in terms: if ":" in term: field, value = term.split(":", 1) if field.lower() == "project": filters.append(JournalEntry.name.ilike(value)) if field.lower() == "version": filters.append(JournalEntry.version.ilike(value)) if field.lower() == "user": filters.append(JournalEntry._submitted_by.like(value)) if field.lower() == "ip": filters.append(JournalEntry.submitted_from.ilike(value)) else: filters.append(JournalEntry.name.ilike(term)) journals_query = journals_query.filter(and_(*filters)) journals = SQLAlchemyORMPage( journals_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"journals": journals, "query": q}
def journals_list(project, request): q = request.params.get("q") project_name = request.matchdict["project_name"] if project_name != project.normalized_name: raise HTTPMovedPermanently( request.current_route_path(project_name=project.normalized_name) ) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None journals_query = ( request.db.query(JournalEntry) .options(joinedload("submitted_by")) .filter(JournalEntry.name == project.name) .order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc()) ) if q: terms = shlex.split(q) filters = [] for term in terms: if ":" in term: field, value = term.split(":", 1) if field.lower() == "version": filters.append(JournalEntry.version.ilike(value)) journals_query = journals_query.filter(or_(*filters)) journals = SQLAlchemyORMPage( journals_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"journals": journals, "project": project, "query": q}
def test_some(self, db_request): check = MalwareCheckFactory.create() for _ in range(10): MalwareVerdictFactory.create(check=check) query = db_request.db.query(MalwareVerdict).order_by( MalwareVerdict.run_date.desc()) verdicts = SQLAlchemyORMPage( query, page=1, items_per_page=25, url_maker=paginate_url_factory(db_request), ) assert views.get_verdicts(db_request) == { "verdicts": verdicts, "check_names": set([check.name]), "classifications": set(["threat", "indeterminate", "benign"]), "confidences": set(["low", "medium", "high"]), }
def email_list(request): q = request.params.get("q") try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None email_query = request.db.query(EmailMessage).order_by( EmailMessage.created.desc(), EmailMessage.id) if q: terms = shlex.split(q) filters = [] for term in terms: if ":" in term: field, value = term.split(":", 1) if field.lower() == "to": filters.append(EmailMessage.to.ilike(value)) if field.lower() == "from": filters.append(EmailMessage.from_.ilike(value)) if field.lower() == "subject": filters.append(EmailMessage.subject.ilike(value)) if field.lower() == "status": filters.append( cast(EmailMessage.status, String).ilike(value)) else: filters.append(EmailMessage.to.ilike(term)) email_query = email_query.filter(or_(*filters)) emails = SQLAlchemyORMPage( email_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"emails": emails, "query": q}
def manage_project_journal(project, request): try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") journals_query = (request.db.query(JournalEntry).options( joinedload("submitted_by")).filter( JournalEntry.name == project.name).order_by( JournalEntry.submitted_date.desc(), JournalEntry.id.desc())) journals = SQLAlchemyORMPage( journals_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) if journals.page_count and page_num > journals.page_count: raise HTTPNotFound return {"project": project, "journals": journals}
def manage_project_history(project, request): try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") events_query = (request.db.query(ProjectEvent).join( ProjectEvent.project).filter( ProjectEvent.project_id == project.id).order_by( ProjectEvent.time.desc())) events = SQLAlchemyORMPage( events_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) if events.page_count and page_num > events.page_count: raise HTTPNotFound return {"project": project, "events": events}
def manage_project_history(project, request): try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") journals_query = ( request.db.query(JournalEntry) .options(joinedload("submitted_by")) .filter(JournalEntry.name == project.name) .order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc()) ) journals = SQLAlchemyORMPage( journals_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) if journals.page_count and page_num > journals.page_count: raise HTTPNotFound return {"project": project, "journals": journals}
def search(request): q = request.params.get("q", '') if q: should = [] for field in SEARCH_FIELDS: kw = {"query": q} if field in SEARCH_BOOSTS: kw["boost"] = SEARCH_BOOSTS[field] should.append(Q("match", **{field: kw})) # Add a prefix query if ``q`` is longer than one character. if len(q) > 1: should.append(Q('prefix', normalized_name=q)) query = request.es.query("dis_max", queries=should) query = query.suggest("name_suggestion", q, term={"field": "name"}) else: query = request.es.query() if request.params.get("o"): sort_key = request.params["o"] if sort_key.startswith("-"): sort = { sort_key[1:]: { "order": "desc", "unmapped_type": "long", }, } else: sort = { sort_key: { "unmapped_type": "long", } } query = query.sort(sort) # Require match to all specified classifiers for classifier in request.params.getall("c"): query = query.filter("terms", classifiers=[classifier]) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") page = ElasticsearchPage( query, page=page_num, url_maker=paginate_url_factory(request), ) if page.page_count and page_num > page.page_count: return HTTPNotFound() available_filters = collections.defaultdict(list) classifiers_q = (request.db.query(Classifier).with_entities( Classifier.classifier).filter( exists([release_classifiers.c.trove_id]).where( release_classifiers.c.trove_id == Classifier.id)).order_by( Classifier.classifier)) for cls in classifiers_q: first, *_ = cls.classifier.split(' :: ') available_filters[first].append(cls.classifier) def filter_key(item): try: return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0] except ValueError: return 1, 0, item[0] return { "page": page, "term": q, "order": request.params.get("o", ''), "available_filters": sorted(available_filters.items(), key=filter_key), "applied_filters": request.params.getall("c"), }
def search(request): q = request.params.get("q", '') q = q.replace("'", '"') if q: bool_query = gather_es_queries(q) query = request.es.query(bool_query) query = query.suggest("name_suggestion", q, term={"field": "name"}) else: query = request.es.query() if request.params.get("o"): sort_key = request.params["o"] if sort_key.startswith("-"): sort = { sort_key[1:]: { "order": "desc", "unmapped_type": "long", }, } else: sort = { sort_key: { "unmapped_type": "long", } } query = query.sort(sort) # Require match to all specified classifiers for classifier in request.params.getall("c"): query = query.filter("terms", classifiers=[classifier]) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") page = ElasticsearchPage( query, page=page_num, url_maker=paginate_url_factory(request), ) if page.page_count and page_num > page.page_count: return HTTPNotFound() available_filters = collections.defaultdict(list) classifiers_q = ( request.db.query(Classifier) .with_entities(Classifier.classifier) .filter(Classifier.deprecated.is_(False)) .filter( exists([release_classifiers.c.trove_id]) .where(release_classifiers.c.trove_id == Classifier.id) ) .order_by(Classifier.classifier) ) for cls in classifiers_q: first, *_ = cls.classifier.split(' :: ') available_filters[first].append(cls.classifier) def filter_key(item): try: return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0] except ValueError: return 1, 0, item[0] request.registry.datadog.histogram('warehouse.views.search.results', page.item_count) return { "page": page, "term": q, "order": request.params.get("o", ''), "available_filters": sorted(available_filters.items(), key=filter_key), "applied_filters": request.params.getall("c"), }
def organization_list(request): if request.flags.enabled(AdminFlagValue.DISABLE_ORGANIZATIONS): raise HTTPNotFound q = request.params.get("q", "") terms = shlex.split(q) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") from None organizations_query = request.db.query(Organization).order_by( Organization.normalized_name) if q: filters = [] for term in terms: # Examples: # - search individual words or "whole phrase" in any field # - name:psf # - organization:python # - url:.org # - description:word # - description:"whole phrase" # - is:approved # - is:declined # - is:submitted # - is:active # - is:inactive try: field, value = term.lower().split(":", 1) except ValueError: field, value = "", term if field == "name": # Add filter for `name` or `normalized_name` fields. filters.append([ Organization.name.ilike(f"%{value}%"), Organization.normalized_name.ilike(f"%{value}%"), ]) elif field == "org" or field == "organization": # Add filter for `display_name` field. filters.append(Organization.display_name.ilike(f"%{value}%")) elif field == "url" or field == "link_url": # Add filter for `link_url` field. filters.append(Organization.link_url.ilike(f"%{value}%")) elif field == "desc" or field == "description": # Add filter for `description` field. filters.append(Organization.description.ilike(f"%{value}%")) elif field == "is": # Add filter for `is_approved` or `is_active` field. if "approved".startswith(value): filters.append( Organization.is_approved == True) # noqa: E712 elif "declined".startswith(value): filters.append( Organization.is_approved == False) # noqa: E712 elif "submitted".startswith(value): filters.append( Organization.is_approved == None) # noqa: E711 elif "active".startswith(value): filters.append( Organization.is_active == True) # noqa: E712 elif "inactive".startswith(value): filters.append( Organization.is_active == False) # noqa: E712 else: # Add filter for any field. filters.append([ Organization.name.ilike(f"%{term}%"), Organization.normalized_name.ilike(f"%{term}%"), Organization.display_name.ilike(f"%{term}%"), Organization.link_url.ilike(f"%{term}%"), Organization.description.ilike(f"%{term}%"), ]) # Use AND to add each filter. Use OR to combine subfilters. for filter_or_subfilters in filters: if isinstance(filter_or_subfilters, list): # Add list of subfilters combined with OR. organizations_query = organizations_query.filter( or_(*filter_or_subfilters)) else: # Add single filter. organizations_query = organizations_query.filter( filter_or_subfilters) organizations = SQLAlchemyORMPage( organizations_query, page=page_num, items_per_page=25, url_maker=paginate_url_factory(request), ) return {"organizations": organizations, "query": q, "terms": terms}
def search(request): metrics = request.find_service(IMetricsService, context=None) q = request.params.get("q", "") q = q.replace("'", '"') if q: bool_query = gather_es_queries(q) query = request.es.query(bool_query) query = query.suggest("name_suggestion", q, term={"field": "name"}) else: query = request.es.query() if request.params.get("o"): sort_key = request.params["o"] if sort_key.startswith("-"): sort = {sort_key[1:]: {"order": "desc", "unmapped_type": "long"}} else: sort = {sort_key: {"unmapped_type": "long"}} query = query.sort(sort) # Require match to all specified classifiers for classifier in request.params.getall("c"): query = query.query("prefix", classifiers=classifier) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") try: page = ElasticsearchPage( query, page=page_num, url_maker=paginate_url_factory(request) ) except elasticsearch.TransportError: metrics.increment("warehouse.views.search.error") raise HTTPServiceUnavailable if page.page_count and page_num > page.page_count: raise HTTPNotFound available_filters = collections.defaultdict(list) classifiers_q = ( request.db.query(Classifier) .with_entities(Classifier.classifier) .filter(Classifier.deprecated.is_(False)) .filter( exists([release_classifiers.c.trove_id]).where( release_classifiers.c.trove_id == Classifier.id ) ) .order_by(Classifier.classifier) ) for cls in classifiers_q: first, *_ = cls.classifier.split(" :: ") available_filters[first].append(cls.classifier) def filter_key(item): try: return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0] except ValueError: return 1, 0, item[0] def form_filters_tree(split_list): """ Takes a list of lists, each of them containing a filter and one of its children. Returns a dictionary, each key being a filter and each value being the filter's children. """ d = {} for l in split_list: current_level = d for part in l: if part not in current_level: current_level[part] = {} current_level = current_level[part] return d def process_available_filters(): """ Processes available filters and returns a list of dictionaries. The value of a key in the dictionary represents its children """ sorted_filters = sorted(available_filters.items(), key=filter_key) output = [] for f in sorted_filters: classifier_list = f[1] split_list = [i.split(" :: ") for i in classifier_list] tree = form_filters_tree(split_list) output.append(tree) return output metrics = request.find_service(IMetricsService, context=None) metrics.histogram("warehouse.views.search.results", page.item_count) return { "page": page, "term": q, "order": request.params.get("o", ""), "available_filters": process_available_filters(), "applied_filters": request.params.getall("c"), }
def search(request): q = request.params.get("q", "") q = q.replace("'", '"') if q: bool_query = gather_es_queries(q) query = request.es.query(bool_query) query = query.suggest("name_suggestion", q, term={"field": "name"}) else: query = request.es.query() if request.params.get("o"): sort_key = request.params["o"] if sort_key.startswith("-"): sort = {sort_key[1:]: {"order": "desc", "unmapped_type": "long"}} else: sort = {sort_key: {"unmapped_type": "long"}} query = query.sort(sort) # Require match to all specified classifiers for classifier in request.params.getall("c"): query = query.filter("terms", classifiers=[classifier]) try: released_range = int(request.params.get("d", 0)) except ValueError: released_range = 0 if released_range: last_week = datetime.datetime.utcnow() - datetime.timedelta( days=released_range) query = query.filter("range", created={"gte": last_week}) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") page = ElasticsearchPage(query, page=page_num, url_maker=paginate_url_factory(request)) if page.page_count and page_num > page.page_count: return HTTPNotFound() by_release_date_filters = ( (0, 'All Time'), (1, 'Today'), (7, 'This week'), (30, 'This month'), (90, 'Last 3 months'), (180, 'Last 6 months'), (365, 'Last year'), (730, 'Last 2 years'), (1095, 'Last 3 years'), (1460, 'Last 4 years'), (2190, 'Last 6 years'), ) available_filters = collections.defaultdict(list) classifiers_q = (request.db.query(Classifier).with_entities( Classifier.classifier).filter(Classifier.deprecated.is_(False)).filter( exists([release_classifiers.c.trove_id]).where( release_classifiers.c.trove_id == Classifier.id)).order_by( Classifier.classifier)) for cls in classifiers_q: first, *_ = cls.classifier.split(" :: ") available_filters[first].append(cls.classifier) def filter_key(item): try: return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0] except ValueError: return 1, 0, item[0] request.registry.datadog.histogram("warehouse.views.search.results", page.item_count) return { "page": page, "term": q, "order": request.params.get("o", ""), "available_filters": sorted(available_filters.items(), key=filter_key), "applied_filters": request.params.getall("c"), "by_release_date_filters": by_release_date_filters, "by_release_date": released_range, }
def search(request): metrics = request.find_service(IMetricsService, context=None) querystring = request.params.get("q", "").replace("'", '"') order = request.params.get("o", "") classifiers = request.params.getall("c") query = get_es_query(request.es, querystring, order, classifiers) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") try: page = ElasticsearchPage(query, page=page_num, url_maker=paginate_url_factory(request)) except elasticsearch.TransportError: metrics.increment("warehouse.views.search.error") raise HTTPServiceUnavailable if page.page_count and page_num > page.page_count: raise HTTPNotFound available_filters = collections.defaultdict(list) classifiers_q = (request.db.query(Classifier).with_entities( Classifier.classifier).filter( exists([release_classifiers.c.trove_id ]).where(release_classifiers.c.trove_id == Classifier.id), Classifier.classifier.notin_(deprecated_classifiers.keys()), ).order_by(Classifier.classifier)) for cls in classifiers_q: first, *_ = cls.classifier.split(" :: ") available_filters[first].append(cls.classifier) def filter_key(item): try: return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0] except ValueError: return 1, 0, item[0] def form_filters_tree(split_list): """ Takes a list of lists, each of them containing a filter and one of its children. Returns a dictionary, each key being a filter and each value being the filter's children. """ d = {} for l in split_list: current_level = d for part in l: if part not in current_level: current_level[part] = {} current_level = current_level[part] return d def process_available_filters(): """ Processes available filters and returns a list of dictionaries. The value of a key in the dictionary represents its children """ sorted_filters = sorted(available_filters.items(), key=filter_key) output = [] for f in sorted_filters: classifier_list = f[1] split_list = [i.split(" :: ") for i in classifier_list] tree = form_filters_tree(split_list) output.append(tree) return output metrics = request.find_service(IMetricsService, context=None) metrics.histogram("warehouse.views.search.results", page.item_count) return { "page": page, "term": querystring, "order": order, "available_filters": process_available_filters(), "applied_filters": request.params.getall("c"), }
def search(request): metrics = request.find_service(IMetricsService, context=None) q = request.params.get("q", "") q = q.replace("'", '"') if q: bool_query = gather_es_queries(q) query = request.es.query(bool_query) query = query.suggest("name_suggestion", q, term={"field": "name"}) else: query = request.es.query() if request.params.get("o"): sort_key = request.params["o"] if sort_key.startswith("-"): sort = {sort_key[1:]: {"order": "desc", "unmapped_type": "long"}} else: sort = {sort_key: {"unmapped_type": "long"}} query = query.sort(sort) # Require match to all specified classifiers for classifier in request.params.getall("c"): query = query.filter("terms", classifiers=[classifier]) try: page_num = int(request.params.get("page", 1)) except ValueError: raise HTTPBadRequest("'page' must be an integer.") try: page = ElasticsearchPage(query, page=page_num, url_maker=paginate_url_factory(request)) except elasticsearch.TransportError: metrics.increment("warehouse.views.search.error") raise HTTPServiceUnavailable if page.page_count and page_num > page.page_count: return HTTPNotFound() available_filters = collections.defaultdict(list) classifiers_q = (request.db.query(Classifier).with_entities( Classifier.classifier).filter(Classifier.deprecated.is_(False)).filter( exists([release_classifiers.c.trove_id]).where( release_classifiers.c.trove_id == Classifier.id)).order_by( Classifier.classifier)) for cls in classifiers_q: first, *_ = cls.classifier.split(" :: ") available_filters[first].append(cls.classifier) def filter_key(item): try: return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0] except ValueError: return 1, 0, item[0] metrics = request.find_service(IMetricsService, context=None) metrics.histogram("warehouse.views.search.results", page.item_count) return { "page": page, "term": q, "order": request.params.get("o", ""), "available_filters": sorted(available_filters.items(), key=filter_key), "applied_filters": request.params.getall("c"), }