def test_tfidf_retriever(): from haystack.retriever.sparse import TfidfRetriever test_docs = [{ "name": "testing the finder 1", "text": "godzilla says hello" }, { "name": "testing the finder 2", "text": "optimus prime says bye" }, { "name": "testing the finder 3", "text": "alien says arghh" }] from haystack.database.memory import InMemoryDocumentStore document_store = InMemoryDocumentStore() document_store.write_documents(test_docs) retriever = TfidfRetriever(document_store) retriever.fit() assert retriever.retrieve("godzilla", top_k=1) == [ Document(id='0', text='godzilla says hello', external_source_id=None, question=None, query_score=None, meta={}) ]
def test_tfidf_retriever(): from haystack.retriever.sparse import TfidfRetriever test_docs = [{ "id": "26f84672c6d7aaeb8e2cd53e9c62d62d", "name": "testing the finder 1", "text": "godzilla says hello" }, { "name": "testing the finder 2", "text": "optimus prime says bye" }, { "name": "testing the finder 3", "text": "alien says arghh" }] from haystack.database.memory import InMemoryDocumentStore document_store = InMemoryDocumentStore() document_store.write_documents(test_docs) retriever = TfidfRetriever(document_store) retriever.fit() doc = retriever.retrieve("godzilla", top_k=1)[0] assert doc.id == "26f84672c6d7aaeb8e2cd53e9c62d62d" assert doc.text == 'godzilla says hello' assert doc.meta == {"name": "testing the finder 1"}
def feed_documents_to_model(model_name="deepset/roberta-base-squad2-covid"): """Feeds documents to model and returns a model ready to make predictions Parameters ---------- model_name : str The path of the model to be selected from HuggingFace By default uses the pretrained version of roBERTa in squad2 and covid articles Returns ------- finder the model to use for predictions """ # Initialize in memory Document Store document_store = InMemoryDocumentStore() # Load articles and format it as dictionary articles = ret.get_data(MANIFEST, ARTICLES_FOLDER, []) dicts_textContent = process_documents(articles) # Store the dictionary with articles content in the Document Store document_store.write_documents(dicts_textContent) # Retriever chooses what is the subset of documents that are relevant # many techniques are possible: for dev purposes TfidfRetriever is faster retriever = TfidfRetriever(document_store=document_store) # Reader provides interface to use the pre trained transformers # by default we're using the roberta reader = FARMReader(model_name_or_path=model_name, use_gpu=False) # The finder retrieves predictions finder = Finder(reader, retriever) return finder
def get_retriever(retriever_type, document_store): if retriever_type == "dpr": retriever = DensePassageRetriever( document_store=document_store, query_embedding_model= "facebook/dpr-question_encoder-single-nq-base", passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base", use_gpu=False, embed_title=True) elif retriever_type == "tfidf": retriever = TfidfRetriever(document_store=document_store) retriever.fit() elif retriever_type == "embedding": retriever = EmbeddingRetriever(document_store=document_store, embedding_model="deepset/sentence_bert", use_gpu=False) elif retriever_type == "elasticsearch": retriever = ElasticsearchRetriever(document_store=document_store) elif retriever_type == "es_filter_only": retriever = ElasticsearchFilterOnlyRetriever( document_store=document_store) else: raise Exception(f"No retriever fixture for '{retriever_type}'") return retriever
def test_finder_get_answers(): test_docs = [{ "name": "testing the finder 1", "text": "testing the finder with pyhton unit test 1", "meta": { "test": "test" } }, { "name": "testing the finder 2", "text": "testing the finder with pyhton unit test 2", "meta": { "test": "test" } }, { "name": "testing the finder 3", "text": "testing the finder with pyhton unit test 3", "meta": { "test": "test" } }] document_store = SQLDocumentStore(url="sqlite:///qa_test.db") document_store.write_documents(test_docs) retriever = TfidfRetriever(document_store=document_store) reader = TransformersReader( model="distilbert-base-uncased-distilled-squad", tokenizer="distilbert-base-uncased", use_gpu=-1) finder = Finder(reader, retriever) prediction = finder.get_answers(question="testing finder", top_k_retriever=10, top_k_reader=5) assert prediction is not None
def load(self): if(self.finder and self.finder2): return if(not self.document_store2): self.document_store2 = FAISSDocumentStore.load( sql_url=sqlUrlFAQ, faiss_file_path='faiss2') # save before load in preprocess self.initSql(url=sqlUrlFAQ, document_store=self.document_store2) # else: # reset session # # self.document_store2.session.close() # super( # FAISSDocumentStore, self.document_store2).__init__(url=sqlUrlFAQ) if(not self.retriever2): self.retriever2 = EmbeddingRetriever(document_store=self.document_store2, embedding_model="sentence_bert-saved", use_gpu=False) if(not self.finder2): self.finder2 = Finder(reader=None, retriever=self.retriever2) if(not self.document_store): self.document_store = SQLDocumentStore(url=sqlUrl) #FAISSDocumentStore.load(faiss_file_path='faiss1', sql_url=sqlUrl) self.initSql(url=sqlUrl, document_store=self.document_store) # else: # reset session # # self.document_store.session.close() # super( # FAISSDocumentStore, self.document_store).__init__(url=sqlUrl) # self.retriever = EmbeddingRetriever( #redice load by sharing the same retriever and set store on fly?? # document_store=self.document_store, embedding_model="sentence_bert-saved", use_gpu=False) if not self.retriever else self.retriever if(not self.retriever): self.retriever = TfidfRetriever(document_store=self.document_store) self.reader = FARMReader(model_name_or_path=modelDir, use_gpu=False, no_ans_boost=0) if not self.reader else self.reader # reader = TransformersReader(model_name_or_path="distilbert-base-uncased-distilled-squad", tokenizer="distilbert-base-uncased", use_gpu=-1) self.finder = Finder( self.reader, self.retriever) if not self.finder else self.finder
def test_finder_get_answers_with_in_memory_store(): test_docs = [{ "text": "testing the finder with pyhton unit test 1", 'meta': { "name": "testing the finder 1", 'url': 'url' } }, { "text": "testing the finder with pyhton unit test 2", 'meta': { "name": "testing the finder 2", 'url': 'url' } }, { "text": "testing the finder with pyhton unit test 3", 'meta': { "name": "testing the finder 3", 'url': 'url' } }] from haystack.database.memory import InMemoryDocumentStore document_store = InMemoryDocumentStore() document_store.write_documents(test_docs) retriever = TfidfRetriever(document_store=document_store) reader = TransformersReader( model="distilbert-base-uncased-distilled-squad", tokenizer="distilbert-base-uncased", use_gpu=-1) finder = Finder(reader, retriever) prediction = finder.get_answers(question="testing finder", top_k_retriever=10, top_k_reader=5) assert prediction is not None
def test_finder_get_answers_single_result(reader, document_store_with_docs): retriever = TfidfRetriever(document_store=document_store_with_docs) finder = Finder(reader, retriever) query = "testing finder" prediction = finder.get_answers(question=query, top_k_retriever=1, top_k_reader=1) assert prediction is not None assert len(prediction["answers"]) == 1
def get_retriever(retriever_name, doc_store): if retriever_name == "elastic": return ElasticsearchRetriever(doc_store) if retriever_name == "tfidf": return TfidfRetriever(doc_store) if retriever_name == "dpr": return DensePassageRetriever(document_store=doc_store, query_embedding_model="facebook/dpr-question_encoder-single-nq-base", passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base", use_gpu=True)
def test_finder_offsets(reader, document_store_with_docs): retriever = TfidfRetriever(document_store=document_store_with_docs) finder = Finder(reader, retriever) prediction = finder.get_answers(question="Who lives in Berlin?", top_k_retriever=10, top_k_reader=5) assert prediction["answers"][0]["offset_start"] == 11 assert prediction["answers"][0]["offset_end"] == 16 start = prediction["answers"][0]["offset_start"] end = prediction["answers"][0]["offset_end"] assert prediction["answers"][0]["context"][start:end] == prediction["answers"][0]["answer"]
def test_finder_offsets(reader, document_store_with_docs): retriever = TfidfRetriever(document_store=document_store_with_docs) finder = Finder(reader, retriever) prediction = finder.get_answers(question="Who lives in Berlin?", top_k_retriever=10, top_k_reader=5) assert prediction["answers"][0]["offset_start"] == 11 #TODO enable again when FARM is upgraded incl. the new offset calc # assert prediction["answers"][0]["offset_end"] == 16 start = prediction["answers"][0]["offset_start"] end = prediction["answers"][0]["offset_end"]
def test_finder_get_answers(reader, document_store_with_docs): retriever = TfidfRetriever(document_store=document_store_with_docs) finder = Finder(reader, retriever) prediction = finder.get_answers(question="Who lives in Berlin?", top_k_retriever=10, top_k_reader=3) assert prediction is not None assert prediction["question"] == "Who lives in Berlin?" assert prediction["answers"][0]["answer"] == "Carla" assert prediction["answers"][0]["probability"] <= 1 assert prediction["answers"][0]["probability"] >= 0 assert prediction["answers"][0]["meta"]["meta_field"] == "test1" assert prediction["answers"][0]["context"] == "My name is Carla and I live in Berlin" assert len(prediction["answers"]) == 3
def get_retriever(retriever_name, doc_store): if retriever_name == "elastic": return ElasticsearchRetriever(doc_store) if retriever_name == "tfidf": return TfidfRetriever(doc_store) if retriever_name == "dpr": return DensePassageRetriever( document_store=doc_store, query_embedding_model= "facebook/dpr-question_encoder-single-nq-base", passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base", use_gpu=True, use_fast_tokenizers=False) if retriever_name == "sentence_transformers": return EmbeddingRetriever(document_store=doc_store, embedding_model="nq-distilbert-base-v1", use_gpu=True, model_format="sentence_transformers")
def retriever(): document_store = read_corpus() retriever = TfidfRetriever(document_store=document_store) return retriever
def tutorial3_basic_qa_pipeline_without_elasticsearch(): # In-Memory Document Store document_store = InMemoryDocumentStore() # or, alternatively, SQLite Document Store # document_store = SQLDocumentStore(url="sqlite:///qa.db") # ## Preprocessing of documents # # Haystack provides a customizable pipeline for: # - converting files into texts # - cleaning texts # - splitting texts # - writing them to a Document Store # In this tutorial, we download Wikipedia articles on Game of Thrones, apply a basic cleaning function, and index # them in Elasticsearch. # Let's first get some documents that we want to query # Here: 517 Wikipedia articles for Game of Thrones doc_dir = "data/article_txt_got" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) # convert files to dicts containing documents that can be indexed to our datastore dicts = convert_files_to_dicts(dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True) # You can optionally supply a cleaning function that is applied to each doc (e.g. to remove footers) # It must take a str as input, and return a str. # Now, let's write the docs to our DB. document_store.write_documents(dicts) # ## Initalize Retriever, Reader, & Finder # # ### Retriever # # Retrievers help narrowing down the scope for the Reader to smaller units of text where # a given question could be answered. # # With InMemoryDocumentStore or SQLDocumentStore, you can use the TfidfRetriever. For more # retrievers, please refer to the tutorial-1. # An in-memory TfidfRetriever based on Pandas dataframes retriever = TfidfRetriever(document_store=document_store) # ### Reader # # A Reader scans the texts returned by retrievers in detail and extracts the k best answers. They are based # on powerful, but slower deep learning models. # # Haystack currently supports Readers based on the frameworks FARM and Transformers. # With both you can either load a local model or one from Hugging Face's model hub (https://huggingface.co/models). # **Here:** a medium sized RoBERTa QA model using a Reader based on # FARM (https://huggingface.co/deepset/roberta-base-squad2) # **Alternatives (Reader):** TransformersReader (leveraging the `pipeline` of the Transformers package) # **Alternatives (Models):** e.g. "distilbert-base-uncased-distilled-squad" (fast) or # "deepset/bert-large-uncased-whole-word-masking-squad2" (good accuracy) # **Hint:** You can adjust the model to return "no answer possible" with the no_ans_boost. # Higher values mean the model prefers "no answer possible". # #### FARMReader # # Load a local model or any of the QA models on # Hugging Face's model hub (https://huggingface.co/models) reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=True) # #### TransformersReader # Alternative: # reader = TransformersReader(model_name_or_path="distilbert-base-uncased-distilled-squad", tokenizer="distilbert-base-uncased", use_gpu=-1) # ### Pipeline # # With a Haystack `Pipeline` you can stick together your building blocks to a search pipeline. # Under the hood, `Pipelines` are Directed Acyclic Graphs (DAGs) that you can easily customize for your own use cases. # To speed things up, Haystack also comes with a few predefined Pipelines. One of them is the `ExtractiveQAPipeline` that combines a retriever and a reader to answer our questions. # You can learn more about `Pipelines` in the [docs](https://haystack.deepset.ai/docs/latest/pipelinesmd). from haystack.pipeline import ExtractiveQAPipeline pipe = ExtractiveQAPipeline(reader, retriever) ## Voilà! Ask a question! prediction = pipe.run(query="Who is the father of Arya Stark?", top_k_retriever=10, top_k_reader=5) # prediction = pipe.run(query="Who created the Dothraki vocabulary?", top_k_reader=5) # prediction = pipe.run(query="Who is the sister of Sansa?", top_k_reader=5) print_answers(prediction, details="minimal")
with open('data.json') as f: data = json.load(f) for article in data: filename = article['url'].split('/')[-1] with open('articles/%s.txt' % filename, 'w') as f: print(article['text'], file=f) # Convert files to dicts # You can optionally supply a cleaning function that is applied to each doc (e.g. to remove footers) # It must take a str as input, and return a str. doc_dir = 'articles' def write_storage(): dicts = convert_files_to_dicts(dir_path=doc_dir, split_paragraphs=True) document_store.write_documents(dicts) write_storage() from haystack.retriever.sparse import TfidfRetriever# ElasticsearchRetriever # retriever = ElasticsearchRetriever(document_store=document_store) retriever = TfidfRetriever(document_store=document_store) reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=False) finder = Finder(reader, retriever) def get_prediction(question, num_retreive, num_read): return finder.get_answers(question=question, top_k_retriever=num_retreive, top_k_reader=num_read)
def tfidf_retriever(inmemory_document_store): return TfidfRetriever(document_store=inmemory_document_store)
from haystack.tokenizer import tokenizer from haystack.utils import print_answers from haystack.document_store.memory import InMemoryDocumentStore from haystack.retriever.sparse import TfidfRetriever print("===============DocumentStore=================") document_store_tfidf = InMemoryDocumentStore() doc_dir_ja = "data/article_txt_got_ja_0" dicts_ja = convert_files_to_dicts(dir_path=doc_dir_ja, clean_func=clean_wiki_text, split_paragraphs=True) print(dicts_ja[0:3]) document_store_tfidf.write_documents(dicts_ja) print("===============Retriever&Reader================") retriever_tfidf = TfidfRetriever(document_store=document_store_tfidf) reader_farm = FARMReader(model_name_or_path="cl-tohoku/bert-base-japanese", use_gpu=True) finder_tfidf_farm = Finder(reader_farm, retriever_tfidf) print("===================question========================") question = "脚本家は誰?" tokenization = tokenizer.FullTokenizer( "./model_sentence_piece/vocab.txt", model_file="./model_sentence_piece/wiki-ja.model", do_lower_case=True) question_tokenize_bySM = tokenization.space_separation(question) prediction = finder_tfidf_farm.get_answers(question=question_tokenize_bySM, top_k_retriever=10, top_k_reader=2)