def feed_documents_to_model(model_name="deepset/roberta-base-squad2-covid"): """Feeds documents to model and returns a model ready to make predictions Parameters ---------- model_name : str The path of the model to be selected from HuggingFace By default uses the pretrained version of roBERTa in squad2 and covid articles Returns ------- finder the model to use for predictions """ # Initialize in memory Document Store document_store = InMemoryDocumentStore() # Load articles and format it as dictionary articles = ret.get_data(MANIFEST, ARTICLES_FOLDER, []) dicts_textContent = process_documents(articles) # Store the dictionary with articles content in the Document Store document_store.write_documents(dicts_textContent) # Retriever chooses what is the subset of documents that are relevant # many techniques are possible: for dev purposes TfidfRetriever is faster retriever = TfidfRetriever(document_store=document_store) # Reader provides interface to use the pre trained transformers # by default we're using the roberta reader = FARMReader(model_name_or_path=model_name, use_gpu=False) # The finder retrieves predictions finder = Finder(reader, retriever) return finder
def test_tfidf_retriever(): from haystack.retriever.sparse import TfidfRetriever test_docs = [ {"id": "26f84672c6d7aaeb8e2cd53e9c62d62d", "name": "testing the finder 1", "text": "godzilla says hello"}, {"name": "testing the finder 2", "text": "optimus prime says bye"}, {"name": "testing the finder 3", "text": "alien says arghh"} ] from haystack.document_store.memory import InMemoryDocumentStore document_store = InMemoryDocumentStore() document_store.write_documents(test_docs) retriever = TfidfRetriever(document_store) retriever.fit() doc = retriever.retrieve("godzilla", top_k=1)[0] assert doc.id == "26f84672c6d7aaeb8e2cd53e9c62d62d" assert doc.text == 'godzilla says hello' assert doc.meta == {"name": "testing the finder 1"}
def read_corpus(): document_store = InMemoryDocumentStore() doc_dir = "Quran" dicts = convert_files_to_dicts(dir_path=doc_dir, split_paragraphs=True) document_store.write_documents(dicts) return document_store
def tutorial3_basic_qa_pipeline_without_elasticsearch(): # In-Memory Document Store document_store = InMemoryDocumentStore() # or, alternatively, SQLite Document Store # document_store = SQLDocumentStore(url="sqlite:///qa.db") # ## Preprocessing of documents # # Haystack provides a customizable pipeline for: # - converting files into texts # - cleaning texts # - splitting texts # - writing them to a Document Store # In this tutorial, we download Wikipedia articles on Game of Thrones, apply a basic cleaning function, and index # them in Elasticsearch. # Let's first get some documents that we want to query # Here: 517 Wikipedia articles for Game of Thrones doc_dir = "data/article_txt_got" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) # convert files to dicts containing documents that can be indexed to our datastore dicts = convert_files_to_dicts(dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True) # You can optionally supply a cleaning function that is applied to each doc (e.g. to remove footers) # It must take a str as input, and return a str. # Now, let's write the docs to our DB. document_store.write_documents(dicts) # ## Initalize Retriever, Reader, & Finder # # ### Retriever # # Retrievers help narrowing down the scope for the Reader to smaller units of text where # a given question could be answered. # # With InMemoryDocumentStore or SQLDocumentStore, you can use the TfidfRetriever. For more # retrievers, please refer to the tutorial-1. # An in-memory TfidfRetriever based on Pandas dataframes retriever = TfidfRetriever(document_store=document_store) # ### Reader # # A Reader scans the texts returned by retrievers in detail and extracts the k best answers. They are based # on powerful, but slower deep learning models. # # Haystack currently supports Readers based on the frameworks FARM and Transformers. # With both you can either load a local model or one from Hugging Face's model hub (https://huggingface.co/models). # **Here:** a medium sized RoBERTa QA model using a Reader based on # FARM (https://huggingface.co/deepset/roberta-base-squad2) # **Alternatives (Reader):** TransformersReader (leveraging the `pipeline` of the Transformers package) # **Alternatives (Models):** e.g. "distilbert-base-uncased-distilled-squad" (fast) or # "deepset/bert-large-uncased-whole-word-masking-squad2" (good accuracy) # **Hint:** You can adjust the model to return "no answer possible" with the no_ans_boost. # Higher values mean the model prefers "no answer possible". # #### FARMReader # # Load a local model or any of the QA models on # Hugging Face's model hub (https://huggingface.co/models) reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=True) # #### TransformersReader # Alternative: # reader = TransformersReader(model_name_or_path="distilbert-base-uncased-distilled-squad", tokenizer="distilbert-base-uncased", use_gpu=-1) # ### Pipeline # # With a Haystack `Pipeline` you can stick together your building blocks to a search pipeline. # Under the hood, `Pipelines` are Directed Acyclic Graphs (DAGs) that you can easily customize for your own use cases. # To speed things up, Haystack also comes with a few predefined Pipelines. One of them is the `ExtractiveQAPipeline` that combines a retriever and a reader to answer our questions. # You can learn more about `Pipelines` in the [docs](https://haystack.deepset.ai/docs/latest/pipelinesmd). from haystack.pipeline import ExtractiveQAPipeline pipe = ExtractiveQAPipeline(reader, retriever) ## Voilà! Ask a question! prediction = pipe.run(query="Who is the father of Arya Stark?", top_k_retriever=10, top_k_reader=5) # prediction = pipe.run(query="Who created the Dothraki vocabulary?", top_k_reader=5) # prediction = pipe.run(query="Who is the sister of Sansa?", top_k_reader=5) print_answers(prediction, details="minimal")
# them in Elasticsearch. # Let's first get some documents that we want to query # Here: 517 Wikipedia articles for Game of Thrones doc_dir = "data/article_txt_got" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) # convert files to dicts containing documents that can be indexed to our datastore dicts = convert_files_to_dicts(dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True) # You can optionally supply a cleaning function that is applied to each doc (e.g. to remove footers) # It must take a str as input, and return a str. # Now, let's write the docs to our DB. document_store.write_documents(dicts) # ## Initalize Retriever, Reader, & Finder # # ### Retriever # # Retrievers help narrowing down the scope for the Reader to smaller units of text where # a given question could be answered. # # With InMemoryDocumentStore or SQLDocumentStore, you can use the TfidfRetriever. For more # retrievers, please refer to the tutorial-1. # An in-memory TfidfRetriever based on Pandas dataframes retriever = TfidfRetriever(document_store=document_store) # ### Reader
from haystack.retriever.sparse import TfidfRetriever from haystack.reader.farm import FARMReader from haystack.pipeline import ExtractiveQAPipeline from collections import Counter from wordcloud import WordCloud # load and transform data and put it in a document store data = pd.read_csv('QA-WordClouds/Womens Clothing E-Commerce Reviews.csv') # convert dataframe to docs docs = [{"text": str(text)} for text in data['Review Text']] print('done') doc_store = InMemoryDocumentStore() doc_store.write_documents(docs) # get haystack pipe with reader and retriever # get retriever retriever = TfidfRetriever(document_store=doc_store) # model for question answering: model_name = 'distilbert-base-cased-distilled-squad' reader = FARMReader(model_name_or_path=model_name, progress_bar=False, return_no_answer=False) # finally the pipe pipe = ExtractiveQAPipeline(reader, retriever) # ask questions and get results from the pipe question = 'How are the colors?'
from haystack.preprocessor.utils import convert_files_to_dicts, fetch_archive_from_http from haystack.reader.farm import FARMReader from haystack.reader.transformers import TransformersReader from haystack.tokenizer import tokenizer from haystack.utils import print_answers from haystack.document_store.memory import InMemoryDocumentStore from haystack.retriever.sparse import TfidfRetriever print("===============DocumentStore=================") document_store_tfidf = InMemoryDocumentStore() doc_dir_ja = "data/article_txt_got_ja_0" dicts_ja = convert_files_to_dicts(dir_path=doc_dir_ja, clean_func=clean_wiki_text, split_paragraphs=True) print(dicts_ja[0:3]) document_store_tfidf.write_documents(dicts_ja) print("===============Retriever&Reader================") retriever_tfidf = TfidfRetriever(document_store=document_store_tfidf) reader_farm = FARMReader(model_name_or_path="cl-tohoku/bert-base-japanese", use_gpu=True) finder_tfidf_farm = Finder(reader_farm, retriever_tfidf) print("===================question========================") question = "脚本家は誰?" tokenization = tokenizer.FullTokenizer( "./model_sentence_piece/vocab.txt", model_file="./model_sentence_piece/wiki-ja.model", do_lower_case=True) question_tokenize_bySM = tokenization.space_separation(question) prediction = finder_tfidf_farm.get_answers(question=question_tokenize_bySM,