Ejemplo n.º 1
0
def blacklist(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    blacklist_query = request.db.query(BlacklistedProject).order_by(
        BlacklistedProject.name
    )

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            filters.append(
                BlacklistedProject.name.ilike(func.normalize_pep426_name(term))
            )

        blacklist_query = blacklist_query.filter(or_(*filters))

    blacklist = SQLAlchemyORMPage(
        blacklist_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"blacklist": blacklist, "query": q}
Ejemplo n.º 2
0
    def test_manually_reviewed_filter(self, db_request, manually_reviewed):
        check = MalwareCheckFactory.create()
        for _ in range(5):
            MalwareVerdictFactory.create(
                check=check, manually_reviewed=bool(manually_reviewed))

        # Create other verdicts to ensure filter works properly
        for _ in range(10):
            MalwareVerdictFactory.create(
                check=check, manually_reviewed=not bool(manually_reviewed))

        db_request.GET["manually_reviewed"] = str(manually_reviewed)

        query = (db_request.db.query(MalwareVerdict).filter(
            MalwareVerdict.manually_reviewed == bool(
                manually_reviewed)).order_by(MalwareVerdict.run_date.desc()))

        verdicts = SQLAlchemyORMPage(
            query,
            page=1,
            items_per_page=25,
            url_maker=paginate_url_factory(db_request),
        )

        response = {
            "verdicts": verdicts,
            "check_names": set([check.name]),
            "classifications": set(["threat", "indeterminate", "benign"]),
            "confidences": set(["low", "medium", "high"]),
        }

        assert views.get_verdicts(db_request) == response
Ejemplo n.º 3
0
def search(request):
    if request.params.get("q"):
        query = request.es.query(
            "multi_match",
            query=request.params["q"],
            fields=[
                "name", "version", "author", "author_email", "maintainer",
                "maintainer_email", "home_page", "license", "summary",
                "description", "keywords", "platform", "download_url",
            ],
        ).suggest(
            name="name_suggestion",
            text=request.params["q"],
            term={"field": "name"}
        )
    else:
        query = request.es.query()

    page = ElasticsearchPage(
        query,
        page=int(request.params.get("page", 1)),
        url_maker=paginate_url_factory(request),
    )

    return {"page": page, "term": request.params.get("q")}
Ejemplo n.º 4
0
def project_list(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    projects_query = request.db.query(Project).order_by(Project.name)

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            filters.append(Project.name.ilike(term))

        projects_query = projects_query.filter(or_(*filters))

    projects = SQLAlchemyORMPage(
        projects_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"projects": projects, "query": q}
Ejemplo n.º 5
0
def email_list(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    email_query = request.db.query(EmailMessage).order_by(
        EmailMessage.created.desc(), EmailMessage.id)

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            filters.append(EmailMessage.to.ilike(term))

        email_query = email_query.filter(or_(*filters))

    emails = SQLAlchemyORMPage(
        email_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"emails": emails, "query": q}
Ejemplo n.º 6
0
def search(request):
    if request.params.get("q"):
        query = request.es.query(
            "multi_match",
            query=request.params["q"],
            fields=[
                "name",
                "version",
                "author",
                "author_email",
                "maintainer",
                "maintainer_email",
                "home_page",
                "license",
                "summary",
                "description",
                "keywords",
                "platform",
                "download_url",
            ],
        )
    else:
        query = request.es.query()

    page = ElasticsearchPage(
        query,
        page=int(request.params.get("page", 1)),
        url_maker=paginate_url_factory(request),
    )

    return {"page": page}
Ejemplo n.º 7
0
def search(request):

    q = request.params.get("q", '')

    if q:
        should = []
        for field in SEARCH_FIELDS:
            kw = {"query": q}
            if field in SEARCH_BOOSTS:
                kw["boost"] = SEARCH_BOOSTS[field]
            should.append(Q("match", **{field: kw}))

        # Add a prefix query if ``q`` is longer than one character.
        if len(q) > 1:
            should.append(Q('prefix', normalized_name=q))

        query = request.es.query("dis_max", queries=should)
        query = query.suggest("name_suggestion", q, term={"field": "name"})
    else:
        query = request.es.query()

    if request.params.get("o"):
        query = query.sort(request.params["o"])

    if request.params.getall("c"):
        query = query.filter("terms", classifiers=request.params.getall("c"))

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    page = ElasticsearchPage(
        query,
        page=page_num,
        url_maker=paginate_url_factory(request),
    )

    if page.page_count and page_num > page.page_count:
        return HTTPNotFound()

    available_filters = collections.defaultdict(list)

    for cls in request.db.query(Classifier).order_by(Classifier.classifier):
        first, *_ = cls.classifier.split(' :: ')
        available_filters[first].append(cls.classifier)

    def filter_key(item):
        try:
            return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
        except ValueError:
            return 1, 0, item[0]

    return {
        "page": page,
        "term": q,
        "order": request.params.get("o", ''),
        "available_filters": sorted(available_filters.items(), key=filter_key),
        "applied_filters": request.params.getall("c"),
    }
Ejemplo n.º 8
0
    def test_classification_filter(self, db_request, classification):
        check = MalwareCheckFactory.create()
        for c in VerdictClassification:
            for _ in range(5):
                MalwareVerdictFactory.create(check=check, classification=c)

        db_request.GET["classification"] = classification

        query = db_request.db.query(MalwareVerdict)
        if classification:
            query = query.filter(MalwareVerdict.classification == classification)
        query = query.order_by(MalwareVerdict.run_date.desc())

        verdicts = SQLAlchemyORMPage(
            query,
            page=1,
            items_per_page=25,
            url_maker=paginate_url_factory(db_request),
        )

        response = {
            "verdicts": verdicts,
            "check_names": set([check.name]),
            "classifications": set(["threat", "indeterminate", "benign"]),
            "confidences": set(["low", "medium", "high"]),
        }
        assert views.get_verdicts(db_request) == response
Ejemplo n.º 9
0
    def test_check_name_filter(self, db_request, check_name):
        for i in range(3):
            check = MalwareCheckFactory.create(name="check%d" % i)
            for _ in range(5):
                MalwareVerdictFactory.create(check=check)

        query = db_request.db.query(MalwareVerdict)
        if check_name:
            query = query.join(MalwareCheck).filter(MalwareCheck.name == check_name)
        query = query.order_by(MalwareVerdict.run_date.desc())

        verdicts = SQLAlchemyORMPage(
            query,
            page=1,
            items_per_page=25,
            url_maker=paginate_url_factory(db_request),
        )

        response = {
            "verdicts": verdicts,
            "check_names": set(["check0", "check1", "check2"]),
            "classifications": set(["threat", "indeterminate", "benign"]),
            "confidences": set(["low", "medium", "high"]),
        }

        db_request.GET["check_name"] = check_name
        assert views.get_verdicts(db_request) == response
Ejemplo n.º 10
0
def user_list(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    users_query = request.db.query(User).order_by(User.username)

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            if ":" in term:
                field, value = term.split(":", 1)
                if field.lower() == "email":
                    filters.append(User.emails.any(Email.email.ilike(value)))
            else:
                filters.append(User.username.ilike(term))

        users_query = users_query.filter(or_(*filters))

    users = SQLAlchemyORMPage(
        users_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"users": users, "query": q}
Ejemplo n.º 11
0
def project_list(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    projects_query = request.db.query(Project).order_by(Project.normalized_name)

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            filters.append(Project.name.ilike(term))

        projects_query = projects_query.filter(or_(*filters))

    projects = SQLAlchemyORMPage(
        projects_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"projects": projects, "query": q}
Ejemplo n.º 12
0
def project_list(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    projects_query = request.db.query(Project).order_by(
        Project.normalized_name)
    exact_match = None

    if q:
        projects_query = projects_query.filter(
            func.ultranormalize_name(Project.name) == func.ultranormalize_name(
                q))

        exact_match = (request.db.query(Project).filter(
            Project.normalized_name == q).one_or_none())

    projects = SQLAlchemyORMPage(
        projects_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"projects": projects, "query": q, "exact_match": exact_match}
Ejemplo n.º 13
0
def user_list(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    users_query = request.db.query(User).order_by(User.username)

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            if ":" in term:
                field, value = term.split(":", 1)
                if field.lower() == "email":
                    filters.append(User.emails.any(Email.email.ilike(value)))
            else:
                filters.append(User.username.ilike(term))

        users_query = users_query.filter(or_(*filters))

    users = SQLAlchemyORMPage(
        users_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"users": users, "query": q}
Ejemplo n.º 14
0
def email_list(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    email_query = request.db.query(EmailMessage).order_by(
        EmailMessage.created.desc(), EmailMessage.id
    )

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            filters.append(EmailMessage.to.ilike(term))

        email_query = email_query.filter(or_(*filters))

    emails = SQLAlchemyORMPage(
        email_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"emails": emails, "query": q}
Ejemplo n.º 15
0
def search(request):

    q = request.params.get("q", '')

    if q:
        should = []
        for field in SEARCH_FIELDS:
            kw = {"query": q}
            if field in SEARCH_BOOSTS:
                kw["boost"] = SEARCH_BOOSTS[field]
            should.append(Q("match", **{field: kw}))

        # Add a prefix query if ``q`` is longer than one character.
        if len(q) > 1:
            should.append(Q('prefix', normalized_name=q))

        query = request.es.query("dis_max", queries=should)
        query = query.suggest("name_suggestion", q, term={"field": "name"})
    else:
        query = request.es.query()

    if request.params.get("o"):
        query = query.sort(request.params["o"])

    if request.params.getall("c"):
        query = query.filter("terms", classifiers=request.params.getall("c"))

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    page = ElasticsearchPage(
        query,
        page=page_num,
        url_maker=paginate_url_factory(request),
    )

    if page.page_count and page_num > page.page_count:
        raise HTTPNotFound

    available_filters = collections.defaultdict(list)

    for cls in request.db.query(Classifier).order_by(Classifier.classifier):
        first, *_ = cls.classifier.split(' :: ')
        available_filters[first].append(cls.classifier)

    return {
        "page": page,
        "term": q,
        "order": request.params.get("o", ''),
        "available_filters": sorted(available_filters.items()),
        "applied_filters": request.params.getall("c"),
    }
Ejemplo n.º 16
0
def search(request):
    if request.params.get("q"):
        query = request.es.query(
            "multi_match",
            query=request.params["q"],
            fields=[
                "author",
                "author_email",
                "description^5",
                "download_url",
                "home_page",
                "keywords^5",
                "license",
                "maintainer",
                "maintainer_email",
                "normalized_name^10",
                "platform",
                "summary^5",
            ],
        ).suggest(name="name_suggestion",
                  text=request.params["q"],
                  term={"field": "name"})
    else:
        query = request.es.query()

    if request.params.get("o"):
        query = query.sort(request.params["o"])

    if request.params.getall("c"):
        query = query.filter('terms', classifiers=request.params.getall("c"))

    page_num = int(request.params.get("page", 1))
    page = ElasticsearchPage(
        query,
        page=page_num,
        url_maker=paginate_url_factory(request),
    )

    if page.page_count and page_num > page.page_count:
        raise HTTPNotFound

    available_filters = collections.defaultdict(list)

    for cls in request.db.query(Classifier).order_by(Classifier.classifier):
        first, *_ = cls.classifier.split(' :: ')
        available_filters[first].append(cls.classifier)

    return {
        "page": page,
        "term": request.params.get("q", ''),
        "order": request.params.get("o", ''),
        "available_filters": sorted(available_filters.items()),
        "applied_filters": request.params.getall("c"),
    }
Ejemplo n.º 17
0
def test_paginate_url(pyramid_request):
    pyramid_request.GET = MultiDict(pyramid_request.GET)
    pyramid_request.GET["foo"] = "bar"

    url = pretend.stub()
    pyramid_request.current_route_path = pretend.call_recorder(lambda _query: url)

    url_maker = paginate.paginate_url_factory(pyramid_request)

    assert url_maker(5) is url
    assert pyramid_request.current_route_path.calls == [
        pretend.call(_query=[("foo", "bar"), ("page", 5)])
    ]
Ejemplo n.º 18
0
def test_paginate_url(pyramid_request):
    pyramid_request.GET = MultiDict(pyramid_request.GET)
    pyramid_request.GET["foo"] = "bar"

    url = pretend.stub()
    pyramid_request.current_route_path = \
        pretend.call_recorder(lambda _query: url)

    url_maker = paginate.paginate_url_factory(pyramid_request)

    assert url_maker(5) is url
    assert pyramid_request.current_route_path.calls == [
        pretend.call(_query=[("foo", "bar"), ("page", 5)]),
    ]
Ejemplo n.º 19
0
def search(request):
    if request.params.get("q"):
        query = request.es.query(
            "multi_match",
            query=request.params["q"],
            fields=[
                "author", "author_email", "description^5", "download_url",
                "home_page", "keywords^5", "license", "maintainer",
                "maintainer_email", "normalized_name^10", "platform",
                "summary^5",
            ],
        ).suggest(
            name="name_suggestion",
            text=request.params["q"],
            term={"field": "name"}
        )
    else:
        query = request.es.query()

    if request.params.get("o"):
        query = query.sort(request.params["o"])

    if request.params.getall("c"):
        query = query.filter('terms', classifiers=request.params.getall("c"))

    page_num = int(request.params.get("page", 1))
    page = ElasticsearchPage(
        query,
        page=page_num,
        url_maker=paginate_url_factory(request),
    )

    if page.page_count and page_num > page.page_count:
        raise HTTPNotFound

    available_filters = collections.defaultdict(list)

    for cls in request.db.query(Classifier).order_by(Classifier.classifier):
        first, *_ = cls.classifier.split(' :: ')
        available_filters[first].append(cls.classifier)

    return {
        "page": page,
        "term": request.params.get("q", ''),
        "order": request.params.get("o", ''),
        "available_filters": sorted(available_filters.items()),
        "applied_filters": request.params.getall("c"),
    }
Ejemplo n.º 20
0
def get_verdicts(request):
    result = {}
    result["check_names"] = set(
        [name for (name, ) in request.db.query(MalwareCheck.name)])
    result["classifications"] = set([c.value for c in VerdictClassification])
    result["confidences"] = set([c.value for c in VerdictConfidence])

    validate_fields(request, result)

    result["verdicts"] = SQLAlchemyORMPage(
        generate_query(request.db, request.params),
        page=int(request.params.get("page", 1)),
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return result
Ejemplo n.º 21
0
def releases_list(project, request):
    q = request.params.get("q")
    project_name = request.matchdict["project_name"]

    if project_name != project.normalized_name:
        raise HTTPMovedPermanently(
            request.current_route_path(
                project_name=project.normalized_name,
            ),
        )

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    releases_query = (request.db.query(Release)
                      .filter(Release.project == project)
                      .order_by(Release._pypi_ordering.desc()))

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            if ":" in term:
                field, value = term.split(":", 1)
                if field.lower() == "version":
                    filters.append(Release.version.ilike(value))

        releases_query = releases_query.filter(or_(*filters))

    releases = SQLAlchemyORMPage(
        releases_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {
        "releases": releases,
        "project": project,
        "query": q,
    }
Ejemplo n.º 22
0
def releases_list(project, request):
    q = request.params.get("q")
    project_name = request.matchdict["project_name"]

    if project_name != project.normalized_name:
        raise HTTPMovedPermanently(
            request.current_route_path(
                project_name=project.normalized_name,
            ),
        )

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    releases_query = (request.db.query(Release)
                      .filter(Release.project == project)
                      .order_by(Release._pypi_ordering.desc()))

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            if ":" in term:
                field, value = term.split(":", 1)
                if field.lower() == "version":
                    filters.append(Release.version.ilike(value))

        releases_query = releases_query.filter(or_(*filters))

    releases = SQLAlchemyORMPage(
        releases_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {
        "releases": releases,
        "project": project,
        "query": q,
    }
Ejemplo n.º 23
0
def journals_list(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    journals_query = (
        request.db.query(JournalEntry)
                  .order_by(
                      JournalEntry.submitted_date.desc(),
                      JournalEntry.id.desc())
    )

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            if ":" in term:
                field, value = term.split(":", 1)
                if field.lower() == "project":
                    filters.append(JournalEntry.name.ilike(value))
                if field.lower() == "version":
                    filters.append(JournalEntry.version.ilike(value))
                if field.lower() == "user":
                    filters.append(JournalEntry._submitted_by.like(value))
                if field.lower() == "ip":
                    filters.append(JournalEntry.submitted_from.ilike(value))
            else:
                filters.append(JournalEntry.name.ilike(term))

        journals_query = journals_query.filter(and_(*filters))

    journals = SQLAlchemyORMPage(
        journals_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"journals": journals, "query": q}
Ejemplo n.º 24
0
def journals_list(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    journals_query = (
        request.db.query(JournalEntry)
        .options(joinedload(JournalEntry.submitted_by))
        .order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc())
    )

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            if ":" in term:
                field, value = term.split(":", 1)
                if field.lower() == "project":
                    filters.append(JournalEntry.name.ilike(value))
                if field.lower() == "version":
                    filters.append(JournalEntry.version.ilike(value))
                if field.lower() == "user":
                    filters.append(JournalEntry._submitted_by.like(value))
                if field.lower() == "ip":
                    filters.append(JournalEntry.submitted_from.ilike(value))
            else:
                filters.append(JournalEntry.name.ilike(term))

        journals_query = journals_query.filter(and_(*filters))

    journals = SQLAlchemyORMPage(
        journals_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"journals": journals, "query": q}
Ejemplo n.º 25
0
def journals_list(project, request):
    q = request.params.get("q")
    project_name = request.matchdict["project_name"]

    if project_name != project.normalized_name:
        raise HTTPMovedPermanently(
            request.current_route_path(project_name=project.normalized_name)
        )

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    journals_query = (
        request.db.query(JournalEntry)
        .options(joinedload("submitted_by"))
        .filter(JournalEntry.name == project.name)
        .order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc())
    )

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            if ":" in term:
                field, value = term.split(":", 1)
                if field.lower() == "version":
                    filters.append(JournalEntry.version.ilike(value))

        journals_query = journals_query.filter(or_(*filters))

    journals = SQLAlchemyORMPage(
        journals_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"journals": journals, "project": project, "query": q}
Ejemplo n.º 26
0
def journals_list(project, request):
    q = request.params.get("q")
    project_name = request.matchdict["project_name"]

    if project_name != project.normalized_name:
        raise HTTPMovedPermanently(
            request.current_route_path(project_name=project.normalized_name)
        )

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    journals_query = (
        request.db.query(JournalEntry)
        .options(joinedload("submitted_by"))
        .filter(JournalEntry.name == project.name)
        .order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc())
    )

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            if ":" in term:
                field, value = term.split(":", 1)
                if field.lower() == "version":
                    filters.append(JournalEntry.version.ilike(value))

        journals_query = journals_query.filter(or_(*filters))

    journals = SQLAlchemyORMPage(
        journals_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"journals": journals, "project": project, "query": q}
Ejemplo n.º 27
0
    def test_some(self, db_request):
        check = MalwareCheckFactory.create()
        for _ in range(10):
            MalwareVerdictFactory.create(check=check)

        query = db_request.db.query(MalwareVerdict).order_by(
            MalwareVerdict.run_date.desc())

        verdicts = SQLAlchemyORMPage(
            query,
            page=1,
            items_per_page=25,
            url_maker=paginate_url_factory(db_request),
        )

        assert views.get_verdicts(db_request) == {
            "verdicts": verdicts,
            "check_names": set([check.name]),
            "classifications": set(["threat", "indeterminate", "benign"]),
            "confidences": set(["low", "medium", "high"]),
        }
Ejemplo n.º 28
0
def email_list(request):
    q = request.params.get("q")

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    email_query = request.db.query(EmailMessage).order_by(
        EmailMessage.created.desc(), EmailMessage.id)

    if q:
        terms = shlex.split(q)

        filters = []
        for term in terms:
            if ":" in term:
                field, value = term.split(":", 1)
                if field.lower() == "to":
                    filters.append(EmailMessage.to.ilike(value))
                if field.lower() == "from":
                    filters.append(EmailMessage.from_.ilike(value))
                if field.lower() == "subject":
                    filters.append(EmailMessage.subject.ilike(value))
                if field.lower() == "status":
                    filters.append(
                        cast(EmailMessage.status, String).ilike(value))
            else:
                filters.append(EmailMessage.to.ilike(term))

        email_query = email_query.filter(or_(*filters))

    emails = SQLAlchemyORMPage(
        email_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"emails": emails, "query": q}
Ejemplo n.º 29
0
def manage_project_journal(project, request):
    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    journals_query = (request.db.query(JournalEntry).options(
        joinedload("submitted_by")).filter(
            JournalEntry.name == project.name).order_by(
                JournalEntry.submitted_date.desc(), JournalEntry.id.desc()))

    journals = SQLAlchemyORMPage(
        journals_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    if journals.page_count and page_num > journals.page_count:
        raise HTTPNotFound

    return {"project": project, "journals": journals}
Ejemplo n.º 30
0
def manage_project_history(project, request):
    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    events_query = (request.db.query(ProjectEvent).join(
        ProjectEvent.project).filter(
            ProjectEvent.project_id == project.id).order_by(
                ProjectEvent.time.desc()))

    events = SQLAlchemyORMPage(
        events_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    if events.page_count and page_num > events.page_count:
        raise HTTPNotFound

    return {"project": project, "events": events}
Ejemplo n.º 31
0
def manage_project_history(project, request):
    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    journals_query = (
        request.db.query(JournalEntry)
        .options(joinedload("submitted_by"))
        .filter(JournalEntry.name == project.name)
        .order_by(JournalEntry.submitted_date.desc(), JournalEntry.id.desc())
    )

    journals = SQLAlchemyORMPage(
        journals_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    if journals.page_count and page_num > journals.page_count:
        raise HTTPNotFound

    return {"project": project, "journals": journals}
Ejemplo n.º 32
0
def search(request):

    q = request.params.get("q", '')

    if q:
        should = []
        for field in SEARCH_FIELDS:
            kw = {"query": q}
            if field in SEARCH_BOOSTS:
                kw["boost"] = SEARCH_BOOSTS[field]
            should.append(Q("match", **{field: kw}))

        # Add a prefix query if ``q`` is longer than one character.
        if len(q) > 1:
            should.append(Q('prefix', normalized_name=q))

        query = request.es.query("dis_max", queries=should)
        query = query.suggest("name_suggestion", q, term={"field": "name"})
    else:
        query = request.es.query()

    if request.params.get("o"):
        sort_key = request.params["o"]
        if sort_key.startswith("-"):
            sort = {
                sort_key[1:]: {
                    "order": "desc",
                    "unmapped_type": "long",
                },
            }
        else:
            sort = {
                sort_key: {
                    "unmapped_type": "long",
                }
            }

        query = query.sort(sort)

    # Require match to all specified classifiers
    for classifier in request.params.getall("c"):
        query = query.filter("terms", classifiers=[classifier])

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    page = ElasticsearchPage(
        query,
        page=page_num,
        url_maker=paginate_url_factory(request),
    )

    if page.page_count and page_num > page.page_count:
        return HTTPNotFound()

    available_filters = collections.defaultdict(list)

    classifiers_q = (request.db.query(Classifier).with_entities(
        Classifier.classifier).filter(
            exists([release_classifiers.c.trove_id]).where(
                release_classifiers.c.trove_id == Classifier.id)).order_by(
                    Classifier.classifier))

    for cls in classifiers_q:
        first, *_ = cls.classifier.split(' :: ')
        available_filters[first].append(cls.classifier)

    def filter_key(item):
        try:
            return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
        except ValueError:
            return 1, 0, item[0]

    return {
        "page": page,
        "term": q,
        "order": request.params.get("o", ''),
        "available_filters": sorted(available_filters.items(), key=filter_key),
        "applied_filters": request.params.getall("c"),
    }
Ejemplo n.º 33
0
def search(request):

    q = request.params.get("q", '')
    q = q.replace("'", '"')

    if q:
        bool_query = gather_es_queries(q)

        query = request.es.query(bool_query)

        query = query.suggest("name_suggestion", q, term={"field": "name"})
    else:
        query = request.es.query()

    if request.params.get("o"):
        sort_key = request.params["o"]
        if sort_key.startswith("-"):
            sort = {
                sort_key[1:]: {
                    "order": "desc",
                    "unmapped_type": "long",
                },
            }
        else:
            sort = {
                sort_key: {
                    "unmapped_type": "long",
                }
            }

        query = query.sort(sort)

    # Require match to all specified classifiers
    for classifier in request.params.getall("c"):
        query = query.filter("terms", classifiers=[classifier])

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    page = ElasticsearchPage(
        query,
        page=page_num,
        url_maker=paginate_url_factory(request),
    )

    if page.page_count and page_num > page.page_count:
        return HTTPNotFound()

    available_filters = collections.defaultdict(list)

    classifiers_q = (
        request.db.query(Classifier)
        .with_entities(Classifier.classifier)
        .filter(Classifier.deprecated.is_(False))
        .filter(
            exists([release_classifiers.c.trove_id])
            .where(release_classifiers.c.trove_id == Classifier.id)
        )
        .order_by(Classifier.classifier)
    )

    for cls in classifiers_q:
        first, *_ = cls.classifier.split(' :: ')
        available_filters[first].append(cls.classifier)

    def filter_key(item):
        try:
            return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
        except ValueError:
            return 1, 0, item[0]

    request.registry.datadog.histogram('warehouse.views.search.results',
                                       page.item_count)

    return {
        "page": page,
        "term": q,
        "order": request.params.get("o", ''),
        "available_filters": sorted(available_filters.items(), key=filter_key),
        "applied_filters": request.params.getall("c"),
    }
Ejemplo n.º 34
0
def organization_list(request):
    if request.flags.enabled(AdminFlagValue.DISABLE_ORGANIZATIONS):
        raise HTTPNotFound

    q = request.params.get("q", "")
    terms = shlex.split(q)

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.") from None

    organizations_query = request.db.query(Organization).order_by(
        Organization.normalized_name)

    if q:
        filters = []
        for term in terms:
            # Examples:
            # - search individual words or "whole phrase" in any field
            # - name:psf
            # - organization:python
            # - url:.org
            # - description:word
            # - description:"whole phrase"
            # - is:approved
            # - is:declined
            # - is:submitted
            # - is:active
            # - is:inactive
            try:
                field, value = term.lower().split(":", 1)
            except ValueError:
                field, value = "", term
            if field == "name":
                # Add filter for `name` or `normalized_name` fields.
                filters.append([
                    Organization.name.ilike(f"%{value}%"),
                    Organization.normalized_name.ilike(f"%{value}%"),
                ])
            elif field == "org" or field == "organization":
                # Add filter for `display_name` field.
                filters.append(Organization.display_name.ilike(f"%{value}%"))
            elif field == "url" or field == "link_url":
                # Add filter for `link_url` field.
                filters.append(Organization.link_url.ilike(f"%{value}%"))
            elif field == "desc" or field == "description":
                # Add filter for `description` field.
                filters.append(Organization.description.ilike(f"%{value}%"))
            elif field == "is":
                # Add filter for `is_approved` or `is_active` field.
                if "approved".startswith(value):
                    filters.append(
                        Organization.is_approved == True)  # noqa: E712
                elif "declined".startswith(value):
                    filters.append(
                        Organization.is_approved == False)  # noqa: E712
                elif "submitted".startswith(value):
                    filters.append(
                        Organization.is_approved == None)  # noqa: E711
                elif "active".startswith(value):
                    filters.append(
                        Organization.is_active == True)  # noqa: E712
                elif "inactive".startswith(value):
                    filters.append(
                        Organization.is_active == False)  # noqa: E712
            else:
                # Add filter for any field.
                filters.append([
                    Organization.name.ilike(f"%{term}%"),
                    Organization.normalized_name.ilike(f"%{term}%"),
                    Organization.display_name.ilike(f"%{term}%"),
                    Organization.link_url.ilike(f"%{term}%"),
                    Organization.description.ilike(f"%{term}%"),
                ])
        # Use AND to add each filter. Use OR to combine subfilters.
        for filter_or_subfilters in filters:
            if isinstance(filter_or_subfilters, list):
                # Add list of subfilters combined with OR.
                organizations_query = organizations_query.filter(
                    or_(*filter_or_subfilters))
            else:
                # Add single filter.
                organizations_query = organizations_query.filter(
                    filter_or_subfilters)

    organizations = SQLAlchemyORMPage(
        organizations_query,
        page=page_num,
        items_per_page=25,
        url_maker=paginate_url_factory(request),
    )

    return {"organizations": organizations, "query": q, "terms": terms}
Ejemplo n.º 35
0
def search(request):
    metrics = request.find_service(IMetricsService, context=None)

    q = request.params.get("q", "")
    q = q.replace("'", '"')

    if q:
        bool_query = gather_es_queries(q)

        query = request.es.query(bool_query)

        query = query.suggest("name_suggestion", q, term={"field": "name"})
    else:
        query = request.es.query()

    if request.params.get("o"):
        sort_key = request.params["o"]
        if sort_key.startswith("-"):
            sort = {sort_key[1:]: {"order": "desc", "unmapped_type": "long"}}
        else:
            sort = {sort_key: {"unmapped_type": "long"}}

        query = query.sort(sort)

    # Require match to all specified classifiers
    for classifier in request.params.getall("c"):
        query = query.query("prefix", classifiers=classifier)

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    try:
        page = ElasticsearchPage(
            query, page=page_num, url_maker=paginate_url_factory(request)
        )
    except elasticsearch.TransportError:
        metrics.increment("warehouse.views.search.error")
        raise HTTPServiceUnavailable

    if page.page_count and page_num > page.page_count:
        raise HTTPNotFound

    available_filters = collections.defaultdict(list)

    classifiers_q = (
        request.db.query(Classifier)
        .with_entities(Classifier.classifier)
        .filter(Classifier.deprecated.is_(False))
        .filter(
            exists([release_classifiers.c.trove_id]).where(
                release_classifiers.c.trove_id == Classifier.id
            )
        )
        .order_by(Classifier.classifier)
    )

    for cls in classifiers_q:
        first, *_ = cls.classifier.split(" :: ")
        available_filters[first].append(cls.classifier)

    def filter_key(item):
        try:
            return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
        except ValueError:
            return 1, 0, item[0]

    def form_filters_tree(split_list):
        """
        Takes a list of lists, each of them containing a filter and
        one of its children.
        Returns a dictionary, each key being a filter and each value being
        the filter's children.
        """
        d = {}
        for l in split_list:
            current_level = d
            for part in l:
                if part not in current_level:
                    current_level[part] = {}
                current_level = current_level[part]
        return d

    def process_available_filters():
        """
        Processes available filters and returns a list of dictionaries.
        The value of a key in the dictionary represents its children
        """
        sorted_filters = sorted(available_filters.items(), key=filter_key)
        output = []
        for f in sorted_filters:
            classifier_list = f[1]
            split_list = [i.split(" :: ") for i in classifier_list]
            tree = form_filters_tree(split_list)
            output.append(tree)
        return output

    metrics = request.find_service(IMetricsService, context=None)
    metrics.histogram("warehouse.views.search.results", page.item_count)

    return {
        "page": page,
        "term": q,
        "order": request.params.get("o", ""),
        "available_filters": process_available_filters(),
        "applied_filters": request.params.getall("c"),
    }
Ejemplo n.º 36
0
def search(request):

    q = request.params.get("q", "")
    q = q.replace("'", '"')

    if q:
        bool_query = gather_es_queries(q)

        query = request.es.query(bool_query)

        query = query.suggest("name_suggestion", q, term={"field": "name"})
    else:
        query = request.es.query()

    if request.params.get("o"):
        sort_key = request.params["o"]
        if sort_key.startswith("-"):
            sort = {sort_key[1:]: {"order": "desc", "unmapped_type": "long"}}
        else:
            sort = {sort_key: {"unmapped_type": "long"}}

        query = query.sort(sort)

    # Require match to all specified classifiers
    for classifier in request.params.getall("c"):
        query = query.filter("terms", classifiers=[classifier])

    try:
        released_range = int(request.params.get("d", 0))
    except ValueError:
        released_range = 0

    if released_range:
        last_week = datetime.datetime.utcnow() - datetime.timedelta(
            days=released_range)
        query = query.filter("range", created={"gte": last_week})

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    page = ElasticsearchPage(query,
                             page=page_num,
                             url_maker=paginate_url_factory(request))

    if page.page_count and page_num > page.page_count:
        return HTTPNotFound()

    by_release_date_filters = (
        (0, 'All Time'),
        (1, 'Today'),
        (7, 'This week'),
        (30, 'This month'),
        (90, 'Last 3 months'),
        (180, 'Last 6 months'),
        (365, 'Last year'),
        (730, 'Last 2 years'),
        (1095, 'Last 3 years'),
        (1460, 'Last 4 years'),
        (2190, 'Last 6 years'),
    )

    available_filters = collections.defaultdict(list)

    classifiers_q = (request.db.query(Classifier).with_entities(
        Classifier.classifier).filter(Classifier.deprecated.is_(False)).filter(
            exists([release_classifiers.c.trove_id]).where(
                release_classifiers.c.trove_id == Classifier.id)).order_by(
                    Classifier.classifier))

    for cls in classifiers_q:
        first, *_ = cls.classifier.split(" :: ")
        available_filters[first].append(cls.classifier)

    def filter_key(item):
        try:
            return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
        except ValueError:
            return 1, 0, item[0]

    request.registry.datadog.histogram("warehouse.views.search.results",
                                       page.item_count)

    return {
        "page": page,
        "term": q,
        "order": request.params.get("o", ""),
        "available_filters": sorted(available_filters.items(), key=filter_key),
        "applied_filters": request.params.getall("c"),
        "by_release_date_filters": by_release_date_filters,
        "by_release_date": released_range,
    }
Ejemplo n.º 37
0
def search(request):
    metrics = request.find_service(IMetricsService, context=None)

    querystring = request.params.get("q", "").replace("'", '"')
    order = request.params.get("o", "")
    classifiers = request.params.getall("c")
    query = get_es_query(request.es, querystring, order, classifiers)

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    try:
        page = ElasticsearchPage(query,
                                 page=page_num,
                                 url_maker=paginate_url_factory(request))
    except elasticsearch.TransportError:
        metrics.increment("warehouse.views.search.error")
        raise HTTPServiceUnavailable

    if page.page_count and page_num > page.page_count:
        raise HTTPNotFound

    available_filters = collections.defaultdict(list)

    classifiers_q = (request.db.query(Classifier).with_entities(
        Classifier.classifier).filter(
            exists([release_classifiers.c.trove_id
                    ]).where(release_classifiers.c.trove_id == Classifier.id),
            Classifier.classifier.notin_(deprecated_classifiers.keys()),
        ).order_by(Classifier.classifier))

    for cls in classifiers_q:
        first, *_ = cls.classifier.split(" :: ")
        available_filters[first].append(cls.classifier)

    def filter_key(item):
        try:
            return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
        except ValueError:
            return 1, 0, item[0]

    def form_filters_tree(split_list):
        """
        Takes a list of lists, each of them containing a filter and
        one of its children.
        Returns a dictionary, each key being a filter and each value being
        the filter's children.
        """
        d = {}
        for l in split_list:
            current_level = d
            for part in l:
                if part not in current_level:
                    current_level[part] = {}
                current_level = current_level[part]
        return d

    def process_available_filters():
        """
        Processes available filters and returns a list of dictionaries.
        The value of a key in the dictionary represents its children
        """
        sorted_filters = sorted(available_filters.items(), key=filter_key)
        output = []
        for f in sorted_filters:
            classifier_list = f[1]
            split_list = [i.split(" :: ") for i in classifier_list]
            tree = form_filters_tree(split_list)
            output.append(tree)
        return output

    metrics = request.find_service(IMetricsService, context=None)
    metrics.histogram("warehouse.views.search.results", page.item_count)

    return {
        "page": page,
        "term": querystring,
        "order": order,
        "available_filters": process_available_filters(),
        "applied_filters": request.params.getall("c"),
    }
Ejemplo n.º 38
0
def search(request):
    metrics = request.find_service(IMetricsService, context=None)

    q = request.params.get("q", "")
    q = q.replace("'", '"')

    if q:
        bool_query = gather_es_queries(q)

        query = request.es.query(bool_query)

        query = query.suggest("name_suggestion", q, term={"field": "name"})
    else:
        query = request.es.query()

    if request.params.get("o"):
        sort_key = request.params["o"]
        if sort_key.startswith("-"):
            sort = {sort_key[1:]: {"order": "desc", "unmapped_type": "long"}}
        else:
            sort = {sort_key: {"unmapped_type": "long"}}

        query = query.sort(sort)

    # Require match to all specified classifiers
    for classifier in request.params.getall("c"):
        query = query.filter("terms", classifiers=[classifier])

    try:
        page_num = int(request.params.get("page", 1))
    except ValueError:
        raise HTTPBadRequest("'page' must be an integer.")

    try:
        page = ElasticsearchPage(query,
                                 page=page_num,
                                 url_maker=paginate_url_factory(request))
    except elasticsearch.TransportError:
        metrics.increment("warehouse.views.search.error")
        raise HTTPServiceUnavailable

    if page.page_count and page_num > page.page_count:
        return HTTPNotFound()

    available_filters = collections.defaultdict(list)

    classifiers_q = (request.db.query(Classifier).with_entities(
        Classifier.classifier).filter(Classifier.deprecated.is_(False)).filter(
            exists([release_classifiers.c.trove_id]).where(
                release_classifiers.c.trove_id == Classifier.id)).order_by(
                    Classifier.classifier))

    for cls in classifiers_q:
        first, *_ = cls.classifier.split(" :: ")
        available_filters[first].append(cls.classifier)

    def filter_key(item):
        try:
            return 0, SEARCH_FILTER_ORDER.index(item[0]), item[0]
        except ValueError:
            return 1, 0, item[0]

    metrics = request.find_service(IMetricsService, context=None)
    metrics.histogram("warehouse.views.search.results", page.item_count)

    return {
        "page": page,
        "term": q,
        "order": request.params.get("o", ""),
        "available_filters": sorted(available_filters.items(), key=filter_key),
        "applied_filters": request.params.getall("c"),
    }