def blacklist(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
blacklist_query = request.db.query(BlacklistedProject).order_by(
BlacklistedProject.name
)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
filters.append(
BlacklistedProject.name.ilike(func.normalize_pep426_name(term))
)
blacklist_query = blacklist_query.filter(or_(*filters))
blacklist = SQLAlchemyORMPage(
blacklist_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"blacklist": blacklist, "query": q}
def test_manually_reviewed_filter(self, db_request, manually_reviewed):
check = MalwareCheckFactory.create()
for _ in range(5):
MalwareVerdictFactory.create(
check=check, manually_reviewed=bool(manually_reviewed))
# Create other verdicts to ensure filter works properly
for _ in range(10):
MalwareVerdictFactory.create(
check=check, manually_reviewed=not bool(manually_reviewed))
db_request.GET["manually_reviewed"] = str(manually_reviewed)
query = (db_request.db.query(MalwareVerdict).filter(
MalwareVerdict.manually_reviewed == bool(
manually_reviewed)).order_by(MalwareVerdict.run_date.desc()))
verdicts = SQLAlchemyORMPage(
query,
page=1,
items_per_page=25,
url_maker=paginate_url_factory(db_request),
)
response = {
"verdicts": verdicts,
"check_names": set([check.name]),
"classifications": set(["threat", "indeterminate", "benign"]),
"confidences": set(["low", "medium", "high"]),
}
assert views.get_verdicts(db_request) == response
def search(request):
if request.params.get("q"):
query = request.es.query(
"multi_match",
query=request.params["q"],
fields=[
"name", "version", "author", "author_email", "maintainer",
"maintainer_email", "home_page", "license", "summary",
"description", "keywords", "platform", "download_url",
],
).suggest(
name="name_suggestion",
text=request.params["q"],
term={"field": "name"}
)
else:
query = request.es.query()
page = ElasticsearchPage(
query,
page=int(request.params.get("page", 1)),
url_maker=paginate_url_factory(request),
)
return {"page": page, "term": request.params.get("q")}
def project_list(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
projects_query = request.db.query(Project).order_by(Project.name)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
filters.append(Project.name.ilike(term))
projects_query = projects_query.filter(or_(*filters))
projects = SQLAlchemyORMPage(
projects_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"projects": projects, "query": q}
def email_list(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
email_query = request.db.query(EmailMessage).order_by(
EmailMessage.created.desc(), EmailMessage.id)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
filters.append(EmailMessage.to.ilike(term))
email_query = email_query.filter(or_(*filters))
emails = SQLAlchemyORMPage(
email_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"emails": emails, "query": q}
def search(request):
if request.params.get("q"):
query = request.es.query(
"multi_match",
query=request.params["q"],
fields=[
"name",
"version",
"author",
"author_email",
"maintainer",
"maintainer_email",
"home_page",
"license",
"summary",
"description",
"keywords",
"platform",
"download_url",
],
)
else:
query = request.es.query()
page = ElasticsearchPage(
query,
page=int(request.params.get("page", 1)),
url_maker=paginate_url_factory(request),
)
return {"page": page}
def search(request):
q = request.params.get("q", '')
if q:
should = []
for field in SEARCH_FIELDS:
kw = {"query": q}
if field in SEARCH_BOOSTS:
kw["boost"] = SEARCH_BOOSTS[field]
should.append(Q("match", **{field: kw}))
# Add a prefix query if ``q`` is longer than one character.
if len(q) > 1:
should.append(Q('prefix', normalized_name=q))
query = request.es.query("dis_max", queries=should)
query = query.suggest("name_suggestion", q, term={"field": "name"})
else:
query = request.es.query()
if request.params.get("o"):
query = query.sort(request.params["o"])
if request.params.getall("c"):
query = query.filter("terms", classifiers=request.params.getall("c"))
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
page = ElasticsearchPage(
query,
page=page_num,
url_maker=paginate_url_factory(request),
)
if page.page_count and page_num > page.page_count:
return HTTPNotFound()
available_filters = collections.defaultdict(list)
for cls in request.db.query(Classifier).order_by(Classifier.classifier):
first, *_ = cls.classifier.split(' :: ')
available_filters[first].append(cls.classifier)
def filter_key(item):
try:
return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
except ValueError:
return 1, 0, item[0]
return {
"page": page,
"term": q,
"order": request.params.get("o", ''),
"available_filters": sorted(available_filters.items(), key=filter_key),
"applied_filters": request.params.getall("c"),
}
def test_classification_filter(self, db_request, classification):
check = MalwareCheckFactory.create()
for c in VerdictClassification:
for _ in range(5):
MalwareVerdictFactory.create(check=check, classification=c)
db_request.GET["classification"] = classification
query = db_request.db.query(MalwareVerdict)
if classification:
query = query.filter(MalwareVerdict.classification == classification)
query = query.order_by(MalwareVerdict.run_date.desc())
verdicts = SQLAlchemyORMPage(
query,
page=1,
items_per_page=25,
url_maker=paginate_url_factory(db_request),
)
response = {
"verdicts": verdicts,
"check_names": set([check.name]),
"classifications": set(["threat", "indeterminate", "benign"]),
"confidences": set(["low", "medium", "high"]),
}
assert views.get_verdicts(db_request) == response
def test_check_name_filter(self, db_request, check_name):
for i in range(3):
check = MalwareCheckFactory.create(name="check%d" % i)
for _ in range(5):
MalwareVerdictFactory.create(check=check)
query = db_request.db.query(MalwareVerdict)
if check_name:
query = query.join(MalwareCheck).filter(MalwareCheck.name == check_name)
query = query.order_by(MalwareVerdict.run_date.desc())
verdicts = SQLAlchemyORMPage(
query,
page=1,
items_per_page=25,
url_maker=paginate_url_factory(db_request),
)
response = {
"verdicts": verdicts,
"check_names": set(["check0", "check1", "check2"]),
"classifications": set(["threat", "indeterminate", "benign"]),
"confidences": set(["low", "medium", "high"]),
}
db_request.GET["check_name"] = check_name
assert views.get_verdicts(db_request) == response
def user_list(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
users_query = request.db.query(User).order_by(User.username)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
if ":" in term:
field, value = term.split(":", 1)
if field.lower() == "email":
filters.append(User.emails.any(Email.email.ilike(value)))
else:
filters.append(User.username.ilike(term))
users_query = users_query.filter(or_(*filters))
users = SQLAlchemyORMPage(
users_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"users": users, "query": q}
def project_list(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
projects_query = request.db.query(Project).order_by(Project.normalized_name)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
filters.append(Project.name.ilike(term))
projects_query = projects_query.filter(or_(*filters))
projects = SQLAlchemyORMPage(
projects_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"projects": projects, "query": q}
def project_list(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
projects_query = request.db.query(Project).order_by(
Project.normalized_name)
exact_match = None
if q:
projects_query = projects_query.filter(
func.ultranormalize_name(Project.name) == func.ultranormalize_name(
q))
exact_match = (request.db.query(Project).filter(
Project.normalized_name == q).one_or_none())
projects = SQLAlchemyORMPage(
projects_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"projects": projects, "query": q, "exact_match": exact_match}
def user_list(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
users_query = request.db.query(User).order_by(User.username)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
if ":" in term:
field, value = term.split(":", 1)
if field.lower() == "email":
filters.append(User.emails.any(Email.email.ilike(value)))
else:
filters.append(User.username.ilike(term))
users_query = users_query.filter(or_(*filters))
users = SQLAlchemyORMPage(
users_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"users": users, "query": q}
def email_list(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
email_query = request.db.query(EmailMessage).order_by(
EmailMessage.created.desc(), EmailMessage.id
)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
filters.append(EmailMessage.to.ilike(term))
email_query = email_query.filter(or_(*filters))
emails = SQLAlchemyORMPage(
email_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"emails": emails, "query": q}
def search(request):
q = request.params.get("q", '')
if q:
should = []
for field in SEARCH_FIELDS:
kw = {"query": q}
if field in SEARCH_BOOSTS:
kw["boost"] = SEARCH_BOOSTS[field]
should.append(Q("match", **{field: kw}))
# Add a prefix query if ``q`` is longer than one character.
if len(q) > 1:
should.append(Q('prefix', normalized_name=q))
query = request.es.query("dis_max", queries=should)
query = query.suggest("name_suggestion", q, term={"field": "name"})
else:
query = request.es.query()
if request.params.get("o"):
query = query.sort(request.params["o"])
if request.params.getall("c"):
query = query.filter("terms", classifiers=request.params.getall("c"))
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
page = ElasticsearchPage(
query,
page=page_num,
url_maker=paginate_url_factory(request),
)
if page.page_count and page_num > page.page_count:
raise HTTPNotFound
available_filters = collections.defaultdict(list)
for cls in request.db.query(Classifier).order_by(Classifier.classifier):
first, *_ = cls.classifier.split(' :: ')
available_filters[first].append(cls.classifier)
return {
"page": page,
"term": q,
"order": request.params.get("o", ''),
"available_filters": sorted(available_filters.items()),
"applied_filters": request.params.getall("c"),
}
def search(request):
if request.params.get("q"):
query = request.es.query(
"multi_match",
query=request.params["q"],
fields=[
"author",
"author_email",
"description^5",
"download_url",
"home_page",
"keywords^5",
"license",
"maintainer",
"maintainer_email",
"normalized_name^10",
"platform",
"summary^5",
],
).suggest(name="name_suggestion",
text=request.params["q"],
term={"field": "name"})
else:
query = request.es.query()
if request.params.get("o"):
query = query.sort(request.params["o"])
if request.params.getall("c"):
query = query.filter('terms', classifiers=request.params.getall("c"))
page_num = int(request.params.get("page", 1))
page = ElasticsearchPage(
query,
page=page_num,
url_maker=paginate_url_factory(request),
)
if page.page_count and page_num > page.page_count:
raise HTTPNotFound
available_filters = collections.defaultdict(list)
for cls in request.db.query(Classifier).order_by(Classifier.classifier):
first, *_ = cls.classifier.split(' :: ')
available_filters[first].append(cls.classifier)
return {
"page": page,
"term": request.params.get("q", ''),
"order": request.params.get("o", ''),
"available_filters": sorted(available_filters.items()),
"applied_filters": request.params.getall("c"),
}
def test_paginate_url(pyramid_request):
pyramid_request.GET = MultiDict(pyramid_request.GET)
pyramid_request.GET["foo"] = "bar"
url = pretend.stub()
pyramid_request.current_route_path = pretend.call_recorder(lambda _query: url)
url_maker = paginate.paginate_url_factory(pyramid_request)
assert url_maker(5) is url
assert pyramid_request.current_route_path.calls == [
pretend.call(_query=[("foo", "bar"), ("page", 5)])
]
def test_paginate_url(pyramid_request):
pyramid_request.GET = MultiDict(pyramid_request.GET)
pyramid_request.GET["foo"] = "bar"
url = pretend.stub()
pyramid_request.current_route_path = \
pretend.call_recorder(lambda _query: url)
url_maker = paginate.paginate_url_factory(pyramid_request)
assert url_maker(5) is url
assert pyramid_request.current_route_path.calls == [
pretend.call(_query=[("foo", "bar"), ("page", 5)]),
]
def search(request):
if request.params.get("q"):
query = request.es.query(
"multi_match",
query=request.params["q"],
fields=[
"author", "author_email", "description^5", "download_url",
"home_page", "keywords^5", "license", "maintainer",
"maintainer_email", "normalized_name^10", "platform",
"summary^5",
],
).suggest(
name="name_suggestion",
text=request.params["q"],
term={"field": "name"}
)
else:
query = request.es.query()
if request.params.get("o"):
query = query.sort(request.params["o"])
if request.params.getall("c"):
query = query.filter('terms', classifiers=request.params.getall("c"))
page_num = int(request.params.get("page", 1))
page = ElasticsearchPage(
query,
page=page_num,
url_maker=paginate_url_factory(request),
)
if page.page_count and page_num > page.page_count:
raise HTTPNotFound
available_filters = collections.defaultdict(list)
for cls in request.db.query(Classifier).order_by(Classifier.classifier):
first, *_ = cls.classifier.split(' :: ')
available_filters[first].append(cls.classifier)
return {
"page": page,
"term": request.params.get("q", ''),
"order": request.params.get("o", ''),
"available_filters": sorted(available_filters.items()),
"applied_filters": request.params.getall("c"),
}
def get_verdicts(request):
result = {}
result["check_names"] = set(
[name for (name, ) in request.db.query(MalwareCheck.name)])
result["classifications"] = set([c.value for c in VerdictClassification])
result["confidences"] = set([c.value for c in VerdictConfidence])
validate_fields(request, result)
result["verdicts"] = SQLAlchemyORMPage(
generate_query(request.db, request.params),
page=int(request.params.get("page", 1)),
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return result
def releases_list(project, request):
q = request.params.get("q")
project_name = request.matchdict["project_name"]
if project_name != project.normalized_name:
raise HTTPMovedPermanently(
request.current_route_path(
project_name=project.normalized_name,
),
)
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
releases_query = (request.db.query(Release)
.filter(Release.project == project)
.order_by(Release._pypi_ordering.desc()))
if q:
terms = shlex.split(q)
filters = []
for term in terms:
if ":" in term:
field, value = term.split(":", 1)
if field.lower() == "version":
filters.append(Release.version.ilike(value))
releases_query = releases_query.filter(or_(*filters))
releases = SQLAlchemyORMPage(
releases_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {
"releases": releases,
"project": project,
"query": q,
}
def releases_list(project, request):
q = request.params.get("q")
project_name = request.matchdict["project_name"]
if project_name != project.normalized_name:
raise HTTPMovedPermanently(
request.current_route_path(
project_name=project.normalized_name,
),
)
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
releases_query = (request.db.query(Release)
.filter(Release.project == project)
.order_by(Release._pypi_ordering.desc()))
if q:
terms = shlex.split(q)
filters = []
for term in terms:
if ":" in term:
field, value = term.split(":", 1)
if field.lower() == "version":
filters.append(Release.version.ilike(value))
releases_query = releases_query.filter(or_(*filters))
releases = SQLAlchemyORMPage(
releases_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {
"releases": releases,
"project": project,
"query": q,
}
def journals_list(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
journals_query = (
request.db.query(JournalEntry)
.order_by(
JournalEntry.submitted_date.desc(),
JournalEntry.id.desc())
)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
if ":" in term:
field, value = term.split(":", 1)
if field.lower() == "project":
filters.append(JournalEntry.name.ilike(value))
if field.lower() == "version":
filters.append(JournalEntry.version.ilike(value))
if field.lower() == "user":
filters.append(JournalEntry._submitted_by.like(value))
if field.lower() == "ip":
filters.append(JournalEntry.submitted_from.ilike(value))
else:
filters.append(JournalEntry.name.ilike(term))
journals_query = journals_query.filter(and_(*filters))
journals = SQLAlchemyORMPage(
journals_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"journals": journals, "query": q}
def journals_list(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
journals_query = (
request.db.query(JournalEntry)
.options(joinedload(JournalEntry.submitted_by))
.order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc())
)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
if ":" in term:
field, value = term.split(":", 1)
if field.lower() == "project":
filters.append(JournalEntry.name.ilike(value))
if field.lower() == "version":
filters.append(JournalEntry.version.ilike(value))
if field.lower() == "user":
filters.append(JournalEntry._submitted_by.like(value))
if field.lower() == "ip":
filters.append(JournalEntry.submitted_from.ilike(value))
else:
filters.append(JournalEntry.name.ilike(term))
journals_query = journals_query.filter(and_(*filters))
journals = SQLAlchemyORMPage(
journals_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"journals": journals, "query": q}
def journals_list(project, request):
q = request.params.get("q")
project_name = request.matchdict["project_name"]
if project_name != project.normalized_name:
raise HTTPMovedPermanently(
request.current_route_path(project_name=project.normalized_name)
)
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
journals_query = (
request.db.query(JournalEntry)
.options(joinedload("submitted_by"))
.filter(JournalEntry.name == project.name)
.order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc())
)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
if ":" in term:
field, value = term.split(":", 1)
if field.lower() == "version":
filters.append(JournalEntry.version.ilike(value))
journals_query = journals_query.filter(or_(*filters))
journals = SQLAlchemyORMPage(
journals_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"journals": journals, "project": project, "query": q}
def journals_list(project, request):
q = request.params.get("q")
project_name = request.matchdict["project_name"]
if project_name != project.normalized_name:
raise HTTPMovedPermanently(
request.current_route_path(project_name=project.normalized_name)
)
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
journals_query = (
request.db.query(JournalEntry)
.options(joinedload("submitted_by"))
.filter(JournalEntry.name == project.name)
.order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc())
)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
if ":" in term:
field, value = term.split(":", 1)
if field.lower() == "version":
filters.append(JournalEntry.version.ilike(value))
journals_query = journals_query.filter(or_(*filters))
journals = SQLAlchemyORMPage(
journals_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"journals": journals, "project": project, "query": q}
def test_some(self, db_request):
check = MalwareCheckFactory.create()
for _ in range(10):
MalwareVerdictFactory.create(check=check)
query = db_request.db.query(MalwareVerdict).order_by(
MalwareVerdict.run_date.desc())
verdicts = SQLAlchemyORMPage(
query,
page=1,
items_per_page=25,
url_maker=paginate_url_factory(db_request),
)
assert views.get_verdicts(db_request) == {
"verdicts": verdicts,
"check_names": set([check.name]),
"classifications": set(["threat", "indeterminate", "benign"]),
"confidences": set(["low", "medium", "high"]),
}
def email_list(request):
q = request.params.get("q")
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
email_query = request.db.query(EmailMessage).order_by(
EmailMessage.created.desc(), EmailMessage.id)
if q:
terms = shlex.split(q)
filters = []
for term in terms:
if ":" in term:
field, value = term.split(":", 1)
if field.lower() == "to":
filters.append(EmailMessage.to.ilike(value))
if field.lower() == "from":
filters.append(EmailMessage.from_.ilike(value))
if field.lower() == "subject":
filters.append(EmailMessage.subject.ilike(value))
if field.lower() == "status":
filters.append(
cast(EmailMessage.status, String).ilike(value))
else:
filters.append(EmailMessage.to.ilike(term))
email_query = email_query.filter(or_(*filters))
emails = SQLAlchemyORMPage(
email_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"emails": emails, "query": q}
def manage_project_journal(project, request):
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
journals_query = (request.db.query(JournalEntry).options(
joinedload("submitted_by")).filter(
JournalEntry.name == project.name).order_by(
JournalEntry.submitted_date.desc(), JournalEntry.id.desc()))
journals = SQLAlchemyORMPage(
journals_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
if journals.page_count and page_num > journals.page_count:
raise HTTPNotFound
return {"project": project, "journals": journals}
def manage_project_history(project, request):
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
events_query = (request.db.query(ProjectEvent).join(
ProjectEvent.project).filter(
ProjectEvent.project_id == project.id).order_by(
ProjectEvent.time.desc()))
events = SQLAlchemyORMPage(
events_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
if events.page_count and page_num > events.page_count:
raise HTTPNotFound
return {"project": project, "events": events}
def manage_project_history(project, request):
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
journals_query = (
request.db.query(JournalEntry)
.options(joinedload("submitted_by"))
.filter(JournalEntry.name == project.name)
.order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc())
)
journals = SQLAlchemyORMPage(
journals_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
if journals.page_count and page_num > journals.page_count:
raise HTTPNotFound
return {"project": project, "journals": journals}
def search(request):
q = request.params.get("q", '')
if q:
should = []
for field in SEARCH_FIELDS:
kw = {"query": q}
if field in SEARCH_BOOSTS:
kw["boost"] = SEARCH_BOOSTS[field]
should.append(Q("match", **{field: kw}))
# Add a prefix query if ``q`` is longer than one character.
if len(q) > 1:
should.append(Q('prefix', normalized_name=q))
query = request.es.query("dis_max", queries=should)
query = query.suggest("name_suggestion", q, term={"field": "name"})
else:
query = request.es.query()
if request.params.get("o"):
sort_key = request.params["o"]
if sort_key.startswith("-"):
sort = {
sort_key[1:]: {
"order": "desc",
"unmapped_type": "long",
},
}
else:
sort = {
sort_key: {
"unmapped_type": "long",
}
}
query = query.sort(sort)
# Require match to all specified classifiers
for classifier in request.params.getall("c"):
query = query.filter("terms", classifiers=[classifier])
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
page = ElasticsearchPage(
query,
page=page_num,
url_maker=paginate_url_factory(request),
)
if page.page_count and page_num > page.page_count:
return HTTPNotFound()
available_filters = collections.defaultdict(list)
classifiers_q = (request.db.query(Classifier).with_entities(
Classifier.classifier).filter(
exists([release_classifiers.c.trove_id]).where(
release_classifiers.c.trove_id == Classifier.id)).order_by(
Classifier.classifier))
for cls in classifiers_q:
first, *_ = cls.classifier.split(' :: ')
available_filters[first].append(cls.classifier)
def filter_key(item):
try:
return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
except ValueError:
return 1, 0, item[0]
return {
"page": page,
"term": q,
"order": request.params.get("o", ''),
"available_filters": sorted(available_filters.items(), key=filter_key),
"applied_filters": request.params.getall("c"),
}
def search(request):
q = request.params.get("q", '')
q = q.replace("'", '"')
if q:
bool_query = gather_es_queries(q)
query = request.es.query(bool_query)
query = query.suggest("name_suggestion", q, term={"field": "name"})
else:
query = request.es.query()
if request.params.get("o"):
sort_key = request.params["o"]
if sort_key.startswith("-"):
sort = {
sort_key[1:]: {
"order": "desc",
"unmapped_type": "long",
},
}
else:
sort = {
sort_key: {
"unmapped_type": "long",
}
}
query = query.sort(sort)
# Require match to all specified classifiers
for classifier in request.params.getall("c"):
query = query.filter("terms", classifiers=[classifier])
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
page = ElasticsearchPage(
query,
page=page_num,
url_maker=paginate_url_factory(request),
)
if page.page_count and page_num > page.page_count:
return HTTPNotFound()
available_filters = collections.defaultdict(list)
classifiers_q = (
request.db.query(Classifier)
.with_entities(Classifier.classifier)
.filter(Classifier.deprecated.is_(False))
.filter(
exists([release_classifiers.c.trove_id])
.where(release_classifiers.c.trove_id == Classifier.id)
)
.order_by(Classifier.classifier)
)
for cls in classifiers_q:
first, *_ = cls.classifier.split(' :: ')
available_filters[first].append(cls.classifier)
def filter_key(item):
try:
return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
except ValueError:
return 1, 0, item[0]
request.registry.datadog.histogram('warehouse.views.search.results',
page.item_count)
return {
"page": page,
"term": q,
"order": request.params.get("o", ''),
"available_filters": sorted(available_filters.items(), key=filter_key),
"applied_filters": request.params.getall("c"),
}
def organization_list(request):
if request.flags.enabled(AdminFlagValue.DISABLE_ORGANIZATIONS):
raise HTTPNotFound
q = request.params.get("q", "")
terms = shlex.split(q)
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.") from None
organizations_query = request.db.query(Organization).order_by(
Organization.normalized_name)
if q:
filters = []
for term in terms:
# Examples:
# - search individual words or "whole phrase" in any field
# - name:psf
# - organization:python
# - url:.org
# - description:word
# - description:"whole phrase"
# - is:approved
# - is:declined
# - is:submitted
# - is:active
# - is:inactive
try:
field, value = term.lower().split(":", 1)
except ValueError:
field, value = "", term
if field == "name":
# Add filter for `name` or `normalized_name` fields.
filters.append([
Organization.name.ilike(f"%{value}%"),
Organization.normalized_name.ilike(f"%{value}%"),
])
elif field == "org" or field == "organization":
# Add filter for `display_name` field.
filters.append(Organization.display_name.ilike(f"%{value}%"))
elif field == "url" or field == "link_url":
# Add filter for `link_url` field.
filters.append(Organization.link_url.ilike(f"%{value}%"))
elif field == "desc" or field == "description":
# Add filter for `description` field.
filters.append(Organization.description.ilike(f"%{value}%"))
elif field == "is":
# Add filter for `is_approved` or `is_active` field.
if "approved".startswith(value):
filters.append(
Organization.is_approved == True) # noqa: E712
elif "declined".startswith(value):
filters.append(
Organization.is_approved == False) # noqa: E712
elif "submitted".startswith(value):
filters.append(
Organization.is_approved == None) # noqa: E711
elif "active".startswith(value):
filters.append(
Organization.is_active == True) # noqa: E712
elif "inactive".startswith(value):
filters.append(
Organization.is_active == False) # noqa: E712
else:
# Add filter for any field.
filters.append([
Organization.name.ilike(f"%{term}%"),
Organization.normalized_name.ilike(f"%{term}%"),
Organization.display_name.ilike(f"%{term}%"),
Organization.link_url.ilike(f"%{term}%"),
Organization.description.ilike(f"%{term}%"),
])
# Use AND to add each filter. Use OR to combine subfilters.
for filter_or_subfilters in filters:
if isinstance(filter_or_subfilters, list):
# Add list of subfilters combined with OR.
organizations_query = organizations_query.filter(
or_(*filter_or_subfilters))
else:
# Add single filter.
organizations_query = organizations_query.filter(
filter_or_subfilters)
organizations = SQLAlchemyORMPage(
organizations_query,
page=page_num,
items_per_page=25,
url_maker=paginate_url_factory(request),
)
return {"organizations": organizations, "query": q, "terms": terms}
def search(request):
metrics = request.find_service(IMetricsService, context=None)
q = request.params.get("q", "")
q = q.replace("'", '"')
if q:
bool_query = gather_es_queries(q)
query = request.es.query(bool_query)
query = query.suggest("name_suggestion", q, term={"field": "name"})
else:
query = request.es.query()
if request.params.get("o"):
sort_key = request.params["o"]
if sort_key.startswith("-"):
sort = {sort_key[1:]: {"order": "desc", "unmapped_type": "long"}}
else:
sort = {sort_key: {"unmapped_type": "long"}}
query = query.sort(sort)
# Require match to all specified classifiers
for classifier in request.params.getall("c"):
query = query.query("prefix", classifiers=classifier)
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
try:
page = ElasticsearchPage(
query, page=page_num, url_maker=paginate_url_factory(request)
)
except elasticsearch.TransportError:
metrics.increment("warehouse.views.search.error")
raise HTTPServiceUnavailable
if page.page_count and page_num > page.page_count:
raise HTTPNotFound
available_filters = collections.defaultdict(list)
classifiers_q = (
request.db.query(Classifier)
.with_entities(Classifier.classifier)
.filter(Classifier.deprecated.is_(False))
.filter(
exists([release_classifiers.c.trove_id]).where(
release_classifiers.c.trove_id == Classifier.id
)
)
.order_by(Classifier.classifier)
)
for cls in classifiers_q:
first, *_ = cls.classifier.split(" :: ")
available_filters[first].append(cls.classifier)
def filter_key(item):
try:
return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
except ValueError:
return 1, 0, item[0]
def form_filters_tree(split_list):
"""
Takes a list of lists, each of them containing a filter and
one of its children.
Returns a dictionary, each key being a filter and each value being
the filter's children.
"""
d = {}
for l in split_list:
current_level = d
for part in l:
if part not in current_level:
current_level[part] = {}
current_level = current_level[part]
return d
def process_available_filters():
"""
Processes available filters and returns a list of dictionaries.
The value of a key in the dictionary represents its children
"""
sorted_filters = sorted(available_filters.items(), key=filter_key)
output = []
for f in sorted_filters:
classifier_list = f[1]
split_list = [i.split(" :: ") for i in classifier_list]
tree = form_filters_tree(split_list)
output.append(tree)
return output
metrics = request.find_service(IMetricsService, context=None)
metrics.histogram("warehouse.views.search.results", page.item_count)
return {
"page": page,
"term": q,
"order": request.params.get("o", ""),
"available_filters": process_available_filters(),
"applied_filters": request.params.getall("c"),
}
def search(request):
q = request.params.get("q", "")
q = q.replace("'", '"')
if q:
bool_query = gather_es_queries(q)
query = request.es.query(bool_query)
query = query.suggest("name_suggestion", q, term={"field": "name"})
else:
query = request.es.query()
if request.params.get("o"):
sort_key = request.params["o"]
if sort_key.startswith("-"):
sort = {sort_key[1:]: {"order": "desc", "unmapped_type": "long"}}
else:
sort = {sort_key: {"unmapped_type": "long"}}
query = query.sort(sort)
# Require match to all specified classifiers
for classifier in request.params.getall("c"):
query = query.filter("terms", classifiers=[classifier])
try:
released_range = int(request.params.get("d", 0))
except ValueError:
released_range = 0
if released_range:
last_week = datetime.datetime.utcnow() - datetime.timedelta(
days=released_range)
query = query.filter("range", created={"gte": last_week})
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
page = ElasticsearchPage(query,
page=page_num,
url_maker=paginate_url_factory(request))
if page.page_count and page_num > page.page_count:
return HTTPNotFound()
by_release_date_filters = (
(0, 'All Time'),
(1, 'Today'),
(7, 'This week'),
(30, 'This month'),
(90, 'Last 3 months'),
(180, 'Last 6 months'),
(365, 'Last year'),
(730, 'Last 2 years'),
(1095, 'Last 3 years'),
(1460, 'Last 4 years'),
(2190, 'Last 6 years'),
)
available_filters = collections.defaultdict(list)
classifiers_q = (request.db.query(Classifier).with_entities(
Classifier.classifier).filter(Classifier.deprecated.is_(False)).filter(
exists([release_classifiers.c.trove_id]).where(
release_classifiers.c.trove_id == Classifier.id)).order_by(
Classifier.classifier))
for cls in classifiers_q:
first, *_ = cls.classifier.split(" :: ")
available_filters[first].append(cls.classifier)
def filter_key(item):
try:
return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
except ValueError:
return 1, 0, item[0]
request.registry.datadog.histogram("warehouse.views.search.results",
page.item_count)
return {
"page": page,
"term": q,
"order": request.params.get("o", ""),
"available_filters": sorted(available_filters.items(), key=filter_key),
"applied_filters": request.params.getall("c"),
"by_release_date_filters": by_release_date_filters,
"by_release_date": released_range,
}
def search(request):
metrics = request.find_service(IMetricsService, context=None)
querystring = request.params.get("q", "").replace("'", '"')
order = request.params.get("o", "")
classifiers = request.params.getall("c")
query = get_es_query(request.es, querystring, order, classifiers)
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
try:
page = ElasticsearchPage(query,
page=page_num,
url_maker=paginate_url_factory(request))
except elasticsearch.TransportError:
metrics.increment("warehouse.views.search.error")
raise HTTPServiceUnavailable
if page.page_count and page_num > page.page_count:
raise HTTPNotFound
available_filters = collections.defaultdict(list)
classifiers_q = (request.db.query(Classifier).with_entities(
Classifier.classifier).filter(
exists([release_classifiers.c.trove_id
]).where(release_classifiers.c.trove_id == Classifier.id),
Classifier.classifier.notin_(deprecated_classifiers.keys()),
).order_by(Classifier.classifier))
for cls in classifiers_q:
first, *_ = cls.classifier.split(" :: ")
available_filters[first].append(cls.classifier)
def filter_key(item):
try:
return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
except ValueError:
return 1, 0, item[0]
def form_filters_tree(split_list):
"""
Takes a list of lists, each of them containing a filter and
one of its children.
Returns a dictionary, each key being a filter and each value being
the filter's children.
"""
d = {}
for l in split_list:
current_level = d
for part in l:
if part not in current_level:
current_level[part] = {}
current_level = current_level[part]
return d
def process_available_filters():
"""
Processes available filters and returns a list of dictionaries.
The value of a key in the dictionary represents its children
"""
sorted_filters = sorted(available_filters.items(), key=filter_key)
output = []
for f in sorted_filters:
classifier_list = f[1]
split_list = [i.split(" :: ") for i in classifier_list]
tree = form_filters_tree(split_list)
output.append(tree)
return output
metrics = request.find_service(IMetricsService, context=None)
metrics.histogram("warehouse.views.search.results", page.item_count)
return {
"page": page,
"term": querystring,
"order": order,
"available_filters": process_available_filters(),
"applied_filters": request.params.getall("c"),
}
def search(request):
metrics = request.find_service(IMetricsService, context=None)
q = request.params.get("q", "")
q = q.replace("'", '"')
if q:
bool_query = gather_es_queries(q)
query = request.es.query(bool_query)
query = query.suggest("name_suggestion", q, term={"field": "name"})
else:
query = request.es.query()
if request.params.get("o"):
sort_key = request.params["o"]
if sort_key.startswith("-"):
sort = {sort_key[1:]: {"order": "desc", "unmapped_type": "long"}}
else:
sort = {sort_key: {"unmapped_type": "long"}}
query = query.sort(sort)
# Require match to all specified classifiers
for classifier in request.params.getall("c"):
query = query.filter("terms", classifiers=[classifier])
try:
page_num = int(request.params.get("page", 1))
except ValueError:
raise HTTPBadRequest("'page' must be an integer.")
try:
page = ElasticsearchPage(query,
page=page_num,
url_maker=paginate_url_factory(request))
except elasticsearch.TransportError:
metrics.increment("warehouse.views.search.error")
raise HTTPServiceUnavailable
if page.page_count and page_num > page.page_count:
return HTTPNotFound()
available_filters = collections.defaultdict(list)
classifiers_q = (request.db.query(Classifier).with_entities(
Classifier.classifier).filter(Classifier.deprecated.is_(False)).filter(
exists([release_classifiers.c.trove_id]).where(
release_classifiers.c.trove_id == Classifier.id)).order_by(
Classifier.classifier))
for cls in classifiers_q:
first, *_ = cls.classifier.split(" :: ")
available_filters[first].append(cls.classifier)
def filter_key(item):
try:
return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
except ValueError:
return 1, 0, item[0]
metrics = request.find_service(IMetricsService, context=None)
metrics.histogram("warehouse.views.search.results", page.item_count)
return {
"page": page,
"term": q,
"order": request.params.get("o", ""),
"available_filters": sorted(available_filters.items(), key=filter_key),
"applied_filters": request.params.getall("c"),
}