def test_you_can_filter_annotations_by_authority(self, factories, index, search):
annotation = factories.Annotation.build(userid="acct:[email protected]")
index(annotation)
response = search.filter("term", authority="example.com").execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_it_can_index_an_annotation_with_no_document(self, factories,
index, get):
annotation = factories.Annotation.build(document=None)
index(annotation)
assert get(annotation.id)["document"] == {}
def test_you_can_make_aggregations_on_tags_raw(self, factories, index, search):
annotation_1 = factories.Annotation.build(
id="test_annotation_id_1", tags=["Hello"]
)
annotation_2 = factories.Annotation.build(
id="test_annotation_id_2", tags=["hello"]
)
index(annotation_1, annotation_2)
tags_aggregation = elasticsearch_dsl.A("terms", field="tags_raw")
search.aggs.bucket("tags_raw_terms", tags_aggregation)
response = search.execute()
tag_bucket_1 = next(
bucket
for bucket in response.aggregations.tags_raw_terms.buckets
if bucket["key"] == "Hello"
)
tag_bucket_2 = next(
bucket
for bucket in response.aggregations.tags_raw_terms.buckets
if bucket["key"] == "hello"
)
assert tag_bucket_1["doc_count"] == 1
assert tag_bucket_2["doc_count"] == 1
def test_you_can_filter_annotations_by_authority(self, factories, index, search):
annotation = factories.Annotation.build(userid="acct:[email protected]")
index(annotation)
response = search.filter("term", authority="example.com").execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_it_can_index_an_annotation_with_no_document(
self, factories, index, get):
annotation = factories.Annotation.build(document=None)
index(annotation)
assert get(annotation.id)["document"] == {}
def test_you_can_filter_annotations_by_group(self, factories, index, search):
annotation = factories.Annotation.build(id="test_annotation_id", groupid="some_group")
index(annotation)
response = search.filter("term", group="some_group").execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_you_can_filter_annotations_by_unicode_text(self, factories, index, search):
annotation = factories.Annotation.build(id="test_annotation_id", text="test ลข ญหฬ")
index(annotation)
response = search.filter("term", text="ลข").execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_it_can_index_an_annotation_with_a_document_with_no_title(
self, factories, index, get):
annotation = factories.Annotation.build(
document=factories.Document.build(title=None), )
index(annotation)
assert "title" not in get(annotation.id)["document"]
def test_it_can_index_an_annotation_with_a_document_with_no_web_uri(
self, factories, index, get_indexed_ann):
annotation = factories.Annotation.build(
document=factories.Document.build(web_uri=None), )
index(annotation)
assert "web_uri" not in get_indexed_ann(annotation.id)["document"]
def test_you_can_filter_annotations_by_creation_time(self, factories, index, search):
before = datetime.datetime.now()
annotation = factories.Annotation.build()
index(annotation)
response = search.filter("range", created={"gte": before}).execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_you_can_filter_annotations_by_id(self, factories, index, search):
annotation = factories.Annotation.build(id="test_ann_id")
index(annotation)
response = search.filter("term", id="test_ann_id").execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_you_can_filter_annotations_by_shared(self, factories, index, search):
annotation = factories.Annotation.build(id="test_annotation_id", shared=False)
index(annotation)
response = search.filter("term", shared=False).execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_you_can_filter_annotations_by_creation_time(self, factories, index, search):
before = datetime.datetime.now()
annotation = factories.Annotation.build()
index(annotation)
response = search.filter("range", created={"gte": before}).execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_you_can_filter_annotations_by_updated_time(self, factories, index, search):
update_time = datetime.datetime.now()
annotation = factories.Annotation.build(id="test_annotation_id", updated=update_time)
index(annotation)
response = search.filter("range", updated={"gte": update_time}).execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_you_can_filter_annotations_by_updated_time(self, factories, index, search):
update_time = datetime.datetime.now()
annotation = factories.Annotation.build(id="test_annotation_id", updated=update_time)
index(annotation)
response = search.filter("range", updated={"gte": update_time}).execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_it_can_index_an_annotation_with_a_document_with_no_web_uri(self, factories, index, get):
annotation = factories.Annotation.build(
document=factories.Document.build(web_uri=None),
)
index(annotation)
assert "web_uri" not in get(annotation.id)["document"]
def test_you_can_filter_annotations_by_thread_ids(self, factories, index, search):
annotation1 = factories.Annotation.build(id="test_annotation_id1")
annotation2 = factories.Annotation.build(id="test_annotation_id2", thread=[annotation1])
index(annotation1, annotation2)
response = search.filter("terms", thread_ids=[annotation1.id]).execute()
assert SearchResponseWithIDs([annotation2.id]) == response
def test_it_indexes_the_annotations_document_title(self, factories,
index, get):
annotation = factories.Annotation.build(
document=factories.Document.build(title="test_document_title"),
)
index(annotation)
assert get(annotation.id)["document"]["title"] == ["test_document_title"]
def test_you_can_filter_annotations_by_uri(self, factories, index, search):
my_uri = 'http://example.com/anything/i/like?ex=something'
annotation = factories.Annotation.build(id="test_annotation_id", target_uri=my_uri)
index(annotation)
response = search.filter("term", uri='example.com/anything/i/like').execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_you_can_filter_annotations_by_uri(self, factories, index, search):
my_uri = 'http://example.com/anything/i/like?ex=something'
annotation = factories.Annotation.build(id="test_annotation_id", target_uri=my_uri)
index(annotation)
response = search.filter("term", uri='example.com/anything/i/like').execute()
assert SearchResponseWithIDs([annotation.id]) == response
def test_it_indexes_the_annotations_document_title(self, factories, index,
get):
annotation = factories.Annotation.build(
document=factories.Document.build(title="test_document_title"), )
index(annotation)
assert get(
annotation.id)["document"]["title"] == ["test_document_title"]
def test_you_can_filter_annotations_by_thread_ids(self, factories, index, search):
annotation1 = factories.Annotation.build(id="test_annotation_id1")
annotation2 = factories.Annotation.build(id="test_annotation_id2", thread=[annotation1])
index(annotation1, annotation2)
response = search.filter("term", thread_ids=[annotation1.id]).execute()
assert SearchResponseWithIDs([annotation2.id]) == response
def test_it_notifies(self, AnnotationTransformEvent, factories, pyramid_request, notify, index, search):
annotation = factories.Annotation.build(userid="acct:[email protected]")
index(annotation)
event = AnnotationTransformEvent.return_value
AnnotationTransformEvent.assert_called_with(pyramid_request, annotation, mock.ANY)
notify.assert_called_once_with(event)
def test_you_can_filter_annotations_by_tags(self, factories, index, search):
annotation = factories.Annotation.build(id="test_annotation_id", tags=["ญหฬ", "tag"])
index(annotation)
response1 = search.filter("term", tags=["ญหฬ"]).execute()
response2 = search.filter("term", tags=["tag"]).execute()
assert SearchResponseWithIDs([annotation.id]) == response1
assert SearchResponseWithIDs([annotation.id]) == response2
def test_annotation_is_marked_deleted(self, es_client, factories,
get_indexed_ann, index):
annotation = factories.Annotation.build(id="test_annotation_id")
index(annotation)
assert 'deleted' not in get_indexed_ann(annotation.id)
h.search.index.delete(es_client, annotation.id)
assert get_indexed_ann(annotation.id).get('deleted') is True
def test_it_can_index_an_annotation_with_a_document_with_no_title(
self, factories, index, get_indexed_ann
):
annotation = factories.Annotation.build(
document=factories.Document.build(title=None)
)
index(annotation)
assert "title" not in get_indexed_ann(annotation.id)["document"]
def test_annotation_ids_are_used_as_elasticsearch_ids(
self, es_client, factories, index):
annotation = factories.Annotation.build()
index(annotation)
result = es_client.conn.get(index=es_client.index,
doc_type=es_client.mapping_type,
id=annotation.id)
assert result["_id"] == annotation.id
def test_you_can_filter_annotations_by_tags(self, factories, index, search):
annotation = factories.Annotation.build(id="test_annotation_id", tags=["ญหฬ", "tag"])
index(annotation)
response1 = search.filter("terms", tags=["ญหฬ"]).execute()
response2 = search.filter("terms", tags=["tag"]).execute()
assert SearchResponseWithIDs([annotation.id]) == response1
assert SearchResponseWithIDs([annotation.id]) == response2
def test_annotation_ids_are_used_as_elasticsearch_ids(
self, es_client, factories, index
):
annotation = factories.Annotation.build()
index(annotation)
result = es_client.conn.get(
index=es_client.index, doc_type=es_client.mapping_type, id=annotation.id
)
assert result["_id"] == annotation.id
def test_annotation_is_marked_deleted(
self, es_client, factories, get_indexed_ann, index
):
annotation = factories.Annotation.build(id="test_annotation_id")
index(annotation)
assert "deleted" not in get_indexed_ann(annotation.id)
h.search.index.delete(es_client, annotation.id)
assert get_indexed_ann(annotation.id).get("deleted") is True
def test_it_notifies(self, AnnotationTransformEvent, factories, pyramid_request, notify, index, search):
annotation = factories.Annotation.build(userid="acct:[email protected]")
index(annotation)
event = AnnotationTransformEvent.return_value
AnnotationTransformEvent.assert_called_with(pyramid_request, annotation, mock.ANY)
# `notify` will be called twice. Once when indexing with ES1, once when
# indexing with ES6.
notify.assert_called_with(event)
def test_annotation_is_marked_deleted(self, es_client, factories, index, search):
annotation = factories.Annotation.build(id="test_annotation_id")
index(annotation)
result = es_client.conn.get(index=es_client.index,
doc_type=es_client.mapping_type,
id=annotation.id)
assert 'deleted' not in result.get('_source')
h.search.index.delete(es_client, annotation.id)
result = es_client.conn.get(index=es_client.index,
doc_type=es_client.mapping_type,
id=annotation.id)
assert result.get('_source').get('deleted') is True
def test_it_indexes_the_annotations_document_web_uri(self, factories,
index, get):
annotation = factories.Annotation.build(
document=factories.Document.build(web_uri="https://example.com/example_article"),
)
index(annotation)
# *Searching* for an annotation by ``annotation.document`` (e.g. by
# document ``title`` or ``web_uri``) isn't enabled. But you can
# retrieve an annotation by ID, or by searching on other field(s), and
# then access its ``document``. Bouncer
# (https://github.com/hypothesis/bouncer) accesses h's Elasticsearch
# index directly and uses this ``document`` field.
assert get(annotation.id)["document"]["web_uri"] == "https://example.com/example_article"
def annotations(self, factories, index):
"""
Add some annotations to Elasticsearch as "noise".
These are annotations that we *don't* expect to show up in search
results. We want some noise in the search index to make sure that the
test search queries are only returning the expected annotations and
not, for example, simply returning *all* annotations.
"""
index(
factories.Annotation.build(),
factories.Annotation.build(),
factories.Annotation.build(),
)
def test_annotation_is_marked_deleted(self, each_es_client, factories, index):
annotation = factories.Annotation.build(id="test_annotation_id")
es_client = each_es_client
index(annotation)
result = es_client.conn.get(index=es_client.index,
doc_type=es_client.mapping_type,
id=annotation.id)
assert 'deleted' not in result.get('_source')
h.search.index.delete(es_client, annotation.id)
result = es_client.conn.get(index=es_client.index,
doc_type=es_client.mapping_type,
id=annotation.id)
assert result.get('_source').get('deleted') is True
def test_you_can_search_within_the_quote(self, factories, index, search, quote, query):
"""Verify that the "TextQuoteSelector" selector is indexed as the "quote" field."""
quote_selector = {
"type": "TextQuoteSelector",
"exact": quote,
"prefix": "something before ",
"suffix": " something after",
}
selectors = [quote_selector]
annotation = factories.Annotation.build(target_selectors=selectors)
index(annotation)
response = search.query("match", quote=query)
assert SearchResponseWithIDs([annotation.id]) == response
def test_it_indexes_the_annotations_document_web_uri(self, factories,
index, get):
annotation = factories.Annotation.build(
document=factories.Document.build(web_uri="https://example.com/example_article"),
)
index(annotation)
# *Searching* for an annotation by ``annotation.document`` (e.g. by
# document ``title`` or ``web_uri``) isn't enabled. But you can
# retrieve an annotation by ID, or by searching on other field(s), and
# then access its ``document``. Bouncer
# (https://github.com/hypothesis/bouncer) accesses h's Elasticsearch
# index directly and uses this ``document`` field.
assert get(annotation.id)["document"]["web_uri"] == "https://example.com/example_article"
def test_you_can_search_within_the_quote(self, factories, index, search, quote, query):
"""Verify that the "TextQuoteSelector" selector is indexed as the "quote" field."""
quote_selector = {
"type": "TextQuoteSelector",
"exact": quote,
"prefix": "something before ",
"suffix": " something after",
}
selectors = [quote_selector]
annotation = factories.Annotation.build(target_selectors=selectors)
index(annotation)
response = search.query("match", quote=query)
assert SearchResponseWithIDs([annotation.id]) == response
def annotations(self, factories, index):
"""
Add some annotations to Elasticsearch as "noise".
These are annotations that we *don't* expect to show up in search
results. We want some noise in the search index to make sure that the
test search queries are only returning the expected annotations and
not, for example, simply returning *all* annotations.
"""
index(
factories.Annotation.build(),
factories.Annotation.build(),
factories.Annotation.build(),
)
def test_it_notifies(self, AnnotationTransformEvent, factories,
pyramid_request, notify, index, search):
annotation = factories.Annotation.build(
userid="acct:[email protected]")
index(annotation)
event = AnnotationTransformEvent.return_value
AnnotationTransformEvent.assert_called_with(pyramid_request,
annotation, mock.ANY)
# `notify` will be called twice. Once when indexing with ES1, once when
# indexing with ES6.
notify.assert_called_with(event)
def test_it_indexes_presented_annotation(self, factories, get_indexed_ann,
index, pyramid_request,
AnnotationSearchIndexPresenter):
annotation = factories.Annotation.build()
presenter = AnnotationSearchIndexPresenter.return_value
presenter.asdict.return_value = {
'id': annotation.id,
'some_other_field': 'a_value'
}
index(annotation)
indexed_ann = get_indexed_ann(annotation.id)
AnnotationSearchIndexPresenter.assert_called_once_with(
annotation, pyramid_request)
assert indexed_ann == presenter.asdict.return_value
def test_you_can_make_aggregations_on_tags_raw(self, factories, index, search):
annotation_1 = factories.Annotation.build(id="test_annotation_id_1", tags=["Hello"])
annotation_2 = factories.Annotation.build(id="test_annotation_id_2", tags=["hello"])
index(annotation_1, annotation_2)
tags_aggregation = elasticsearch1_dsl.A('terms', field='tags_raw')
search.aggs.bucket('tags_raw_terms', tags_aggregation)
response = search.execute()
tag_bucket_1 = next(bucket for bucket in response.aggregations.tags_raw_terms.buckets if bucket["key"] == "Hello")
tag_bucket_2 = next(bucket for bucket in response.aggregations.tags_raw_terms.buckets if bucket["key"] == "hello")
assert tag_bucket_1["doc_count"] == 1
assert tag_bucket_2["doc_count"] == 1
def test_you_can_make_aggregations_on_user_raw(self, factories, index, search):
annotation_1 = factories.Annotation.build(userid="acct:[email protected]")
annotation_2 = factories.Annotation.build(userid="acct:[email protected]")
index(annotation_1, annotation_2)
user_aggregation = elasticsearch1_dsl.A('terms', field='user_raw')
search.aggs.bucket('user_raw_terms', user_aggregation)
response = search.execute()
user_bucket_1 = next(bucket for bucket in response.aggregations.user_raw_terms.buckets
if bucket["key"] == "acct:[email protected]")
user_bucket_2 = next(bucket for bucket in response.aggregations.user_raw_terms.buckets
if bucket["key"] == "acct:[email protected]")
assert user_bucket_1["doc_count"] == 1
assert user_bucket_2["doc_count"] == 1
def test_you_can_filter_annotations_by_hidden(
self, AnnotationSearchIndexPresenter, factories, index, search):
annotation1 = factories.Annotation.build()
annotation2 = factories.Annotation.build()
presenter = AnnotationSearchIndexPresenter.return_value
presenter.asdict.return_value = {'id': annotation1.id, 'hidden': True}
index(annotation1)
presenter = AnnotationSearchIndexPresenter.return_value
presenter.asdict.return_value = {'id': annotation2.id, 'hidden': False}
index(annotation2)
response = search.filter("term", hidden=True).execute()
assert SearchResponseWithIDs([annotation1.id]) == response
def test_you_can_make_aggregations_on_user_raw(self, factories, index, search):
annotation_1 = factories.Annotation.build(userid="acct:[email protected]")
annotation_2 = factories.Annotation.build(userid="acct:[email protected]")
index(annotation_1, annotation_2)
agg = aggregate(search)
user_aggregation = agg('terms', field='user_raw')
search.aggs.bucket('user_raw_terms', user_aggregation)
response = search.execute()
user_bucket_1 = next(bucket for bucket in response.aggregations.user_raw_terms.buckets
if bucket["key"] == "acct:[email protected]")
user_bucket_2 = next(bucket for bucket in response.aggregations.user_raw_terms.buckets
if bucket["key"] == "acct:[email protected]")
assert user_bucket_1["doc_count"] == 1
assert user_bucket_2["doc_count"] == 1
def test_you_can_filter_annotations_by_hidden(
self, AnnotationSearchIndexPresenter, factories, index, search
):
annotation1 = factories.Annotation.build()
annotation2 = factories.Annotation.build()
presenter = AnnotationSearchIndexPresenter.return_value
presenter.asdict.return_value = {"id": annotation1.id, "hidden": True}
index(annotation1)
presenter = AnnotationSearchIndexPresenter.return_value
presenter.asdict.return_value = {"id": annotation2.id, "hidden": False}
index(annotation2)
response = search.filter("term", hidden=True).execute()
assert SearchResponseWithIDs([annotation1.id]) == response
def test_it_notifies(
self,
AnnotationTransformEvent,
factories,
pyramid_request,
notify,
index,
search,
):
annotation = factories.Annotation.build(userid="acct:[email protected]")
index(annotation)
event = AnnotationTransformEvent.return_value
AnnotationTransformEvent.assert_called_once_with(
pyramid_request, annotation, mock.ANY
)
notify.assert_called_once_with(event)
def test_it_indexes_presented_annotation(
self,
factories,
get_indexed_ann,
index,
pyramid_request,
AnnotationSearchIndexPresenter,
):
annotation = factories.Annotation.build()
presenter = AnnotationSearchIndexPresenter.return_value
presenter.asdict.return_value = {
"id": annotation.id,
"some_other_field": "a_value",
}
index(annotation)
indexed_ann = get_indexed_ann(annotation.id)
AnnotationSearchIndexPresenter.assert_called_once_with(
annotation, pyramid_request
)
assert indexed_ann == presenter.asdict.return_value
def _Annotation(**kwargs):
annotation = factories.Annotation.build(**kwargs)
index(annotation)
return annotation
def _Annotation(**kwargs):
annotation = factories.Annotation.build(**kwargs)
index(annotation)
return annotation