def feed_documents_to_model(model_name="deepset/roberta-base-squad2-covid"): """Feeds documents to model and returns a model ready to make predictions Parameters ---------- model_name : str The path of the model to be selected from HuggingFace By default uses the pretrained version of roBERTa in squad2 and covid articles Returns ------- finder the model to use for predictions """ # Initialize in memory Document Store document_store = InMemoryDocumentStore() # Load articles and format it as dictionary articles = ret.get_data(MANIFEST, ARTICLES_FOLDER, []) dicts_textContent = process_documents(articles) # Store the dictionary with articles content in the Document Store document_store.write_documents(dicts_textContent) # Retriever chooses what is the subset of documents that are relevant # many techniques are possible: for dev purposes TfidfRetriever is faster retriever = TfidfRetriever(document_store=document_store) # Reader provides interface to use the pre trained transformers # by default we're using the roberta reader = FARMReader(model_name_or_path=model_name, use_gpu=False) # The finder retrieves predictions finder = Finder(reader, retriever) return finder
def get_document_store(document_store_type, embedding_field="embedding"): if document_store_type == "sql": document_store = SQLDocumentStore(url="sqlite://", index="haystack_test") elif document_store_type == "memory": document_store = InMemoryDocumentStore( return_embedding=True, embedding_field=embedding_field, index="haystack_test" ) elif document_store_type == "elasticsearch": # make sure we start from a fresh index client = Elasticsearch() client.indices.delete(index='haystack_test*', ignore=[404]) document_store = ElasticsearchDocumentStore( index="haystack_test", return_embedding=True, embedding_field=embedding_field ) elif document_store_type == "faiss": document_store = FAISSDocumentStore( sql_url="sqlite://", return_embedding=True, embedding_field=embedding_field, index="haystack_test", ) return document_store elif document_store_type == "milvus": document_store = MilvusDocumentStore( sql_url="sqlite://", return_embedding=True, embedding_field=embedding_field, index="haystack_test", ) return document_store else: raise Exception(f"No document store fixture for '{document_store_type}'") return document_store
def get_document_store(document_store_type, similarity='dot_product'): """ TODO This method is taken from test/conftest.py but maybe should be within Haystack. Perhaps a class method of DocStore that just takes string for type of DocStore""" if document_store_type == "sql": if os.path.exists("haystack_test.db"): os.remove("haystack_test.db") document_store = SQLDocumentStore(url="sqlite:///haystack_test.db") assert document_store.get_document_count() == 0 elif document_store_type == "memory": document_store = InMemoryDocumentStore() elif document_store_type == "elasticsearch": # make sure we start from a fresh index client = Elasticsearch() client.indices.delete(index='haystack_test*', ignore=[404]) document_store = ElasticsearchDocumentStore(index="eval_document", similarity=similarity, timeout=3000) elif document_store_type in ("milvus_flat", "milvus_hnsw"): if document_store_type == "milvus_flat": index_type = IndexType.FLAT index_param = None search_param = None elif document_store_type == "milvus_hnsw": index_type = IndexType.HNSW index_param = {"M": 64, "efConstruction": 80} search_param = {"ef": 20} document_store = MilvusDocumentStore(similarity=similarity, index_type=index_type, index_param=index_param, search_param=search_param) assert document_store.get_document_count(index="eval_document") == 0 elif document_store_type in ("faiss_flat", "faiss_hnsw"): if document_store_type == "faiss_flat": index_type = "Flat" elif document_store_type == "faiss_hnsw": index_type = "HNSW" status = subprocess.run(['docker rm -f haystack-postgres'], shell=True) time.sleep(1) status = subprocess.run([ 'docker run --name haystack-postgres -p 5432:5432 -e POSTGRES_PASSWORD=password -d postgres' ], shell=True) time.sleep(6) status = subprocess.run([ 'docker exec haystack-postgres psql -U postgres -c "CREATE DATABASE haystack;"' ], shell=True) time.sleep(1) document_store = FAISSDocumentStore( sql_url="postgresql://*****:*****@localhost:5432/haystack", faiss_index_factory_str=index_type, similarity=similarity) assert document_store.get_document_count() == 0 else: raise Exception( f"No document store fixture for '{document_store_type}'") return document_store
def test_tfidf_retriever(): from haystack.retriever.sparse import TfidfRetriever test_docs = [ {"id": "26f84672c6d7aaeb8e2cd53e9c62d62d", "name": "testing the finder 1", "text": "godzilla says hello"}, {"name": "testing the finder 2", "text": "optimus prime says bye"}, {"name": "testing the finder 3", "text": "alien says arghh"} ] from haystack.document_store.memory import InMemoryDocumentStore document_store = InMemoryDocumentStore() document_store.write_documents(test_docs) retriever = TfidfRetriever(document_store) retriever.fit() doc = retriever.retrieve("godzilla", top_k=1)[0] assert doc.id == "26f84672c6d7aaeb8e2cd53e9c62d62d" assert doc.text == 'godzilla says hello' assert doc.meta == {"name": "testing the finder 1"}
def get_document_store(document_store_type, embedding_dim=768, embedding_field="embedding"): if document_store_type == "sql": document_store = SQLDocumentStore(url="sqlite://", index="haystack_test") elif document_store_type == "memory": document_store = InMemoryDocumentStore(return_embedding=True, embedding_dim=embedding_dim, embedding_field=embedding_field, index="haystack_test") elif document_store_type == "elasticsearch": # make sure we start from a fresh index client = Elasticsearch() client.indices.delete(index='haystack_test*', ignore=[404]) document_store = ElasticsearchDocumentStore( index="haystack_test", return_embedding=True, embedding_dim=embedding_dim, embedding_field=embedding_field) elif document_store_type == "faiss": document_store = FAISSDocumentStore( vector_dim=embedding_dim, sql_url="sqlite://", return_embedding=True, embedding_field=embedding_field, index="haystack_test", ) return document_store elif document_store_type == "milvus": document_store = MilvusDocumentStore( vector_dim=embedding_dim, sql_url="sqlite://", return_embedding=True, embedding_field=embedding_field, index="haystack_test", ) _, collections = document_store.milvus_server.list_collections() for collection in collections: if collection.startswith("haystack_test"): document_store.milvus_server.drop_collection(collection) return document_store elif document_store_type == "weaviate": document_store = WeaviateDocumentStore( weaviate_url="http://localhost:8080", index="Haystacktest") document_store.weaviate_client.schema.delete_all() document_store._create_schema_and_index_if_not_exist() return document_store else: raise Exception( f"No document store fixture for '{document_store_type}'") return document_store
def get_document_store(document_store_type, es_similarity='cosine'): """ TODO This method is taken from test/conftest.py but maybe should be within Haystack. Perhaps a class method of DocStore that just takes string for type of DocStore""" if document_store_type == "sql": if os.path.exists("haystack_test.db"): os.remove("haystack_test.db") document_store = SQLDocumentStore(url="sqlite:///haystack_test.db") elif document_store_type == "memory": document_store = InMemoryDocumentStore() elif document_store_type == "elasticsearch": # make sure we start from a fresh index client = Elasticsearch() client.indices.delete(index='haystack_test*', ignore=[404]) document_store = ElasticsearchDocumentStore(index="eval_document", similarity=es_similarity) elif document_store_type in ("faiss_flat", "faiss_hnsw"): if document_store_type == "faiss_flat": index_type = "Flat" elif document_store_type == "faiss_hnsw": index_type = "HNSW" #TEMP FIX for issue with deleting docs # status = subprocess.run( # ['docker rm -f haystack-postgres'], # shell=True) # time.sleep(3) # try: # document_store = FAISSDocumentStore(sql_url="postgresql://*****:*****@localhost:5432/haystack", # faiss_index_factory_str=index_type) # except: # Launch a postgres instance & create empty DB # logger.info("Didn't find Postgres. Start a new instance...") status = subprocess.run(['docker rm -f haystack-postgres'], shell=True) time.sleep(1) status = subprocess.run([ 'docker run --name haystack-postgres -p 5432:5432 -e POSTGRES_PASSWORD=password -d postgres' ], shell=True) time.sleep(3) status = subprocess.run([ 'docker exec -it haystack-postgres psql -U postgres -c "CREATE DATABASE haystack;"' ], shell=True) time.sleep(1) document_store = FAISSDocumentStore( sql_url="postgresql://*****:*****@localhost:5432/haystack", faiss_index_factory_str=index_type) else: raise Exception( f"No document store fixture for '{document_store_type}'") assert document_store.get_document_count() == 0 return document_store
def get_document_store(document_store_type): if document_store_type == "sql": if os.path.exists("haystack_test.db"): os.remove("haystack_test.db") document_store = SQLDocumentStore(url="sqlite:///haystack_test.db") elif document_store_type == "memory": document_store = InMemoryDocumentStore() elif document_store_type == "elasticsearch": # make sure we start from a fresh index client = Elasticsearch() client.indices.delete(index='haystack_test*', ignore=[404]) document_store = ElasticsearchDocumentStore(index="haystack_test") elif document_store_type == "faiss": if os.path.exists("haystack_test_faiss.db"): os.remove("haystack_test_faiss.db") document_store = FAISSDocumentStore( sql_url="sqlite:///haystack_test_faiss.db") else: raise Exception( f"No document store fixture for '{document_store_type}'") return document_store
def read_corpus(): document_store = InMemoryDocumentStore() doc_dir = "Quran" dicts = convert_files_to_dicts(dir_path=doc_dir, split_paragraphs=True) document_store.write_documents(dicts) return document_store
def tutorial9_dpr_training(): # Training Your Own "Dense Passage Retrieval" Model # Here are some imports that we'll need from haystack.retriever.dense import DensePassageRetriever from haystack.preprocessor.utils import fetch_archive_from_http from haystack.document_store.memory import InMemoryDocumentStore # Download original DPR data # WARNING: the train set is 7.4GB and the dev set is 800MB doc_dir = "data/dpr_training/" s3_url_train = "https://dl.fbaipublicfiles.com/dpr/data/retriever/biencoder-nq-train.json.gz" s3_url_dev = "https://dl.fbaipublicfiles.com/dpr/data/retriever/biencoder-nq-dev.json.gz" fetch_archive_from_http(s3_url_train, output_dir=doc_dir + "train/") fetch_archive_from_http(s3_url_dev, output_dir=doc_dir + "dev/") ## Option 1: Training DPR from Scratch # Here are the variables to specify our training data, the models that we use to initialize DPR # and the directory where we'll be saving the model doc_dir = "data/dpr_training/" train_filename = "train/biencoder-nq-train.json" dev_filename = "dev/biencoder-nq-dev.json" query_model = "bert-base-uncased" passage_model = "bert-base-uncased" save_dir = "../saved_models/dpr" # ## Option 2: Finetuning DPR # # # Here are the variables you might want to use instead of the set above # # in order to perform pretraining # # doc_dir = "PATH_TO_YOUR_DATA_DIR" # train_filename = "TRAIN_FILENAME" # dev_filename = "DEV_FILENAME" # # query_model = "facebook/dpr-question_encoder-single-nq-base" # passage_model = "facebook/dpr-ctx_encoder-single-nq-base" # # save_dir = "..saved_models/dpr" ## Initialize DPR model retriever = DensePassageRetriever( document_store=InMemoryDocumentStore(), query_embedding_model=query_model, passage_embedding_model=passage_model, max_seq_len_query=64, max_seq_len_passage=256 ) # Start training our model and save it when it is finished retriever.train( data_dir=doc_dir, train_filename=train_filename, dev_filename=dev_filename, test_filename=dev_filename, n_epochs=1, batch_size=4, grad_acc_steps=4, save_dir=save_dir, evaluate_every=3000, embed_title=True, num_positives=1, num_hard_negatives=1 ) ## Loading reloaded_retriever = DensePassageRetriever.load(load_dir=save_dir, document_store=None)
def tutorial3_basic_qa_pipeline_without_elasticsearch(): # In-Memory Document Store document_store = InMemoryDocumentStore() # or, alternatively, SQLite Document Store # document_store = SQLDocumentStore(url="sqlite:///qa.db") # ## Preprocessing of documents # # Haystack provides a customizable pipeline for: # - converting files into texts # - cleaning texts # - splitting texts # - writing them to a Document Store # In this tutorial, we download Wikipedia articles on Game of Thrones, apply a basic cleaning function, and index # them in Elasticsearch. # Let's first get some documents that we want to query # Here: 517 Wikipedia articles for Game of Thrones doc_dir = "data/article_txt_got" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) # convert files to dicts containing documents that can be indexed to our datastore dicts = convert_files_to_dicts(dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True) # You can optionally supply a cleaning function that is applied to each doc (e.g. to remove footers) # It must take a str as input, and return a str. # Now, let's write the docs to our DB. document_store.write_documents(dicts) # ## Initalize Retriever, Reader, & Finder # # ### Retriever # # Retrievers help narrowing down the scope for the Reader to smaller units of text where # a given question could be answered. # # With InMemoryDocumentStore or SQLDocumentStore, you can use the TfidfRetriever. For more # retrievers, please refer to the tutorial-1. # An in-memory TfidfRetriever based on Pandas dataframes retriever = TfidfRetriever(document_store=document_store) # ### Reader # # A Reader scans the texts returned by retrievers in detail and extracts the k best answers. They are based # on powerful, but slower deep learning models. # # Haystack currently supports Readers based on the frameworks FARM and Transformers. # With both you can either load a local model or one from Hugging Face's model hub (https://huggingface.co/models). # **Here:** a medium sized RoBERTa QA model using a Reader based on # FARM (https://huggingface.co/deepset/roberta-base-squad2) # **Alternatives (Reader):** TransformersReader (leveraging the `pipeline` of the Transformers package) # **Alternatives (Models):** e.g. "distilbert-base-uncased-distilled-squad" (fast) or # "deepset/bert-large-uncased-whole-word-masking-squad2" (good accuracy) # **Hint:** You can adjust the model to return "no answer possible" with the no_ans_boost. # Higher values mean the model prefers "no answer possible". # #### FARMReader # # Load a local model or any of the QA models on # Hugging Face's model hub (https://huggingface.co/models) reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=True) # #### TransformersReader # Alternative: # reader = TransformersReader(model_name_or_path="distilbert-base-uncased-distilled-squad", tokenizer="distilbert-base-uncased", use_gpu=-1) # ### Pipeline # # With a Haystack `Pipeline` you can stick together your building blocks to a search pipeline. # Under the hood, `Pipelines` are Directed Acyclic Graphs (DAGs) that you can easily customize for your own use cases. # To speed things up, Haystack also comes with a few predefined Pipelines. One of them is the `ExtractiveQAPipeline` that combines a retriever and a reader to answer our questions. # You can learn more about `Pipelines` in the [docs](https://haystack.deepset.ai/docs/latest/pipelinesmd). from haystack.pipeline import ExtractiveQAPipeline pipe = ExtractiveQAPipeline(reader, retriever) ## Voilà! Ask a question! prediction = pipe.run(query="Who is the father of Arya Stark?", top_k_retriever=10, top_k_reader=5) # prediction = pipe.run(query="Who created the Dothraki vocabulary?", top_k_reader=5) # prediction = pipe.run(query="Who is the sister of Sansa?", top_k_reader=5) print_answers(prediction, details="minimal")
def get_retriever_for_training( query_model="facebook/dpr-question_encoder-single-nq-base", passage_model="facebook/dpr-ctx_encoder-single-nq-base"): return get_retriever(InMemoryDocumentStore(), query_model=query_model, passage_model=passage_model)
def inmemory_document_store(): return InMemoryDocumentStore(return_embedding=True)
# You can use an `InMemoryDocumentStore` or a `SQLDocumentStore`(with SQLite) as the document store. # # If you are interested in more feature-rich Elasticsearch, then please refer to the Tutorial 1. from haystack import Finder from haystack.document_store.memory import InMemoryDocumentStore from haystack.document_store.sql import SQLDocumentStore from haystack.preprocessor.cleaning import clean_wiki_text from haystack.preprocessor.utils import convert_files_to_dicts, fetch_archive_from_http from haystack.reader.farm import FARMReader from haystack.reader.transformers import TransformersReader from haystack.retriever.sparse import TfidfRetriever from haystack.utils import print_answers # In-Memory Document Store document_store = InMemoryDocumentStore() # or, alternatively, SQLite Document Store # document_store = SQLDocumentStore(url="sqlite:///qa.db") # ## Preprocessing of documents # # Haystack provides a customizable pipeline for: # - converting files into texts # - cleaning texts # - splitting texts # - writing them to a Document Store # In this tutorial, we download Wikipedia articles on Game of Thrones, apply a basic cleaning function, and index # them in Elasticsearch. # Let's first get some documents that we want to query
from haystack.document_store.memory import InMemoryDocumentStore from haystack.retriever.sparse import TfidfRetriever from haystack.reader.farm import FARMReader from haystack.pipeline import ExtractiveQAPipeline from collections import Counter from wordcloud import WordCloud # load and transform data and put it in a document store data = pd.read_csv('QA-WordClouds/Womens Clothing E-Commerce Reviews.csv') # convert dataframe to docs docs = [{"text": str(text)} for text in data['Review Text']] print('done') doc_store = InMemoryDocumentStore() doc_store.write_documents(docs) # get haystack pipe with reader and retriever # get retriever retriever = TfidfRetriever(document_store=doc_store) # model for question answering: model_name = 'distilbert-base-cased-distilled-squad' reader = FARMReader(model_name_or_path=model_name, progress_bar=False, return_no_answer=False) # finally the pipe pipe = ExtractiveQAPipeline(reader, retriever) # ask questions and get results from the pipe question = 'How are the colors?'
from haystack import Finder from haystack.preprocessor.cleaning import clean_wiki_text from haystack.preprocessor.utils import convert_files_to_dicts, fetch_archive_from_http from haystack.reader.farm import FARMReader from haystack.reader.transformers import TransformersReader from haystack.tokenizer import tokenizer from haystack.utils import print_answers from haystack.document_store.memory import InMemoryDocumentStore from haystack.retriever.sparse import TfidfRetriever print("===============DocumentStore=================") document_store_tfidf = InMemoryDocumentStore() doc_dir_ja = "data/article_txt_got_ja_0" dicts_ja = convert_files_to_dicts(dir_path=doc_dir_ja, clean_func=clean_wiki_text, split_paragraphs=True) print(dicts_ja[0:3]) document_store_tfidf.write_documents(dicts_ja) print("===============Retriever&Reader================") retriever_tfidf = TfidfRetriever(document_store=document_store_tfidf) reader_farm = FARMReader(model_name_or_path="cl-tohoku/bert-base-japanese", use_gpu=True) finder_tfidf_farm = Finder(reader_farm, retriever_tfidf) print("===================question========================") question = "脚本家は誰?" tokenization = tokenizer.FullTokenizer( "./model_sentence_piece/vocab.txt", model_file="./model_sentence_piece/wiki-ja.model", do_lower_case=True)
retriever = ElasticsearchRetriever(document_store=document_store) reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=False) finder = Finder(reader, retriever) prediction = finder.get_answers(question="Who is the education minister", top_k_retriever=10, top_k_reader=5) print_answers(prediction, details="minimal") """Question answering without Elastic search""" from haystack.document_store.memory import InMemoryDocumentStore document_store = InMemoryDocumentStore() doc_dir = "data/article_txt_got" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) dicts = convert_files_to_dicts(dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True) print(dicts[:3]) document_store.write_documents(dicts) from haystack.retriever.sparse import TfidfRetriever retriever = TfidfRetriever(document_store=document_store) reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=False)