def test_faiss_update_docs(document_store, index_buffer_size):
# adjust buffer size
document_store.index_buffer_size = index_buffer_size
# initial write
document_store.write_documents(DOCUMENTS)
# do the update
retriever = DensePassageRetriever(
document_store=document_store,
query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
use_gpu=False,
embed_title=True,
remove_sep_tok_from_untitled_passages=True)
document_store.update_embeddings(retriever=retriever)
documents_indexed = document_store.get_all_documents()
# test if correct vectors are associated with docs
for i, doc in enumerate(documents_indexed):
original_doc = [d for d in DOCUMENTS if d["text"] == doc.text][0]
updated_embedding = retriever.embed_passages(
[Document.from_dict(original_doc)])
stored_emb = document_store.faiss_index.reconstruct(
int(doc.meta["vector_id"]))
# compare original input vec with stored one (ignore extra dim added by hnsw)
assert np.allclose(updated_embedding, stored_emb, rtol=0.01)
# test document correctness
check_data_correctness(documents_indexed, DOCUMENTS)
def test_dpr_passage_encoder():
from haystack.retriever.dense import DensePassageRetriever
passage = ["Let's encode this one"]
retriever = DensePassageRetriever(document_store=None,
embedding_model="dpr-bert-base-nq",
gpu=False)
emb = retriever.embed_passages(passage)[0]
assert (emb.shape[0] == 768)
assert (emb[0] - 0.52872 < 0.001)
def test_dpr_inmemory_retrieval():
document_store = InMemoryDocumentStore()
documents = [
{
'name':
'0',
'text':
"""Aaron Aaron ( or ; ""Ahärôn"") is a prophet, high priest, and the brother of Moses in the Abrahamic religions. Knowledge of Aaron, along with his brother Moses, comes exclusively from religious texts, such as the Bible and Quran. The Hebrew Bible relates that, unlike Moses, who grew up in the Egyptian royal court, Aaron and his elder sister Miriam remained with their kinsmen in the eastern border-land of Egypt (Goshen). When Moses first confronted the Egyptian king about the Israelites, Aaron served as his brother's spokesman (""prophet"") to the Pharaoh. Part of the Law (Torah) that Moses received from"""
},
{
'name':
'1',
'text':
"""Schopenhauer, describing him as an ultimately shallow thinker: ""Schopenhauer has quite a crude mind ... where real depth starts, his comes to an end."" His friend Bertrand Russell had a low opinion on the philosopher, and attacked him in his famous ""History of Western Philosophy"" for hypocritically praising asceticism yet not acting upon it. On the opposite isle of Russell on the foundations of mathematics, the Dutch mathematician L. E. J. Brouwer incorporated the ideas of Kant and Schopenhauer in intuitionism, where mathematics is considered a purely mental activity, instead of an analytic activity wherein objective properties of reality are"""
},
{
'name':
'2',
'text':
"""Democratic Republic of the Congo to the south. Angola's capital, Luanda, lies on the Atlantic coast in the northwest of the country. Angola, although located in a tropical zone, has a climate that is not characterized for this region, due to the confluence of three factors: As a result, Angola's climate is characterized by two seasons: rainfall from October to April and drought, known as ""Cacimbo"", from May to August, drier, as the name implies, and with lower temperatures. On the other hand, while the coastline has high rainfall rates, decreasing from North to South and from to , with"""
},
]
retriever = DensePassageRetriever(document_store=document_store,
embedding_model="dpr-bert-base-nq",
use_gpu=False)
embedded = []
for doc in documents:
embedding = retriever.embed_passages([doc['text']])[0]
doc['embedding'] = embedding
embedded.append(doc)
assert (embedding.shape[0] == 768)
assert (embedding[0] - 0.52872 < 0.001)
document_store.write_documents(embedded)
res = retriever.retrieve(query="Which philosopher attacked Schopenhauer?")
assert res[0].text == documents[1]["text"]
