import json import time import concurrent import pylcs from haystack.retriever.sparse import ElasticsearchRetriever from tqdm import tqdm import dpr.experiments.document_store as document_store_utils from dpr.retrievers.dataset.NQDataset import NQDataset document_store = document_store_utils.get_elastic_document_store() print(document_store.get_document_count()) retriever = ElasticsearchRetriever(document_store=document_store) def retrieve_inner(context, result): retrieve = retriever.retrieve(context, top_k=1) # print('searched for:', context) text = retrieve[0].text # print('found:', text) lcs = pylcs.lcs2(text, context) if lcs >= min(len(text), len(context)) * 0.5: result.append(context) bar = tqdm(total=307_000) took = 0 raise Exception #do not overwrite this fail with open('new_nq_dev.json', 'w') as new_nq_file:
def test_elasticsearch_custom_query(elasticsearch_fixture): client = Elasticsearch() client.indices.delete(index="haystack_test_custom", ignore=[404]) document_store = ElasticsearchDocumentStore( index="haystack_test_custom", text_field="custom_text_field", embedding_field="custom_embedding_field") documents = [ { "text": "test_1", "meta": { "year": "2019" } }, { "text": "test_2", "meta": { "year": "2020" } }, { "text": "test_3", "meta": { "year": "2021" } }, { "text": "test_4", "meta": { "year": "2021" } }, { "text": "test_5", "meta": { "year": "2021" } }, ] document_store.write_documents(documents) # test custom "terms" query retriever = ElasticsearchRetriever( document_store=document_store, custom_query=""" { "size": 10, "query": { "bool": { "should": [{ "multi_match": {"query": ${query}, "type": "most_fields", "fields": ["text"]}}], "filter": [{"terms": {"year": ${years}}}]}}}""", ) results = retriever.retrieve(query="test", filters={"years": ["2020", "2021"]}) assert len(results) == 4 # test custom "term" query retriever = ElasticsearchRetriever( document_store=document_store, custom_query=""" { "size": 10, "query": { "bool": { "should": [{ "multi_match": {"query": ${query}, "type": "most_fields", "fields": ["text"]}}], "filter": [{"term": {"year": ${years}}}]}}}""", ) results = retriever.retrieve(query="test", filters={"years": "2021"}) assert len(results) == 3
# Download evaluation data, which is a subset of Natural Questions development set containing 50 documents doc_dir = "../data/nq" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset.json.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) # Connect to Elasticsearch document_store = ElasticsearchDocumentStore(host="localhost", username="", password="", index="document", create_index=False) # Add evaluation data to Elasticsearch database if LAUNCH_ELASTICSEARCH: document_store.add_eval_data("../data/nq/nq_dev_subset.json") else: logger.warning("Since we already have a running ES instance we should not index the same documents again." "If you still want to do this call: 'document_store.add_eval_data('../data/nq/nq_dev_subset.json')' manually ") # Initialize Retriever retriever = ElasticsearchRetriever(document_store=document_store) # Initialize Reader reader = FARMReader("deepset/roberta-base-squad2") # Initialize Finder which sticks together Reader and Retriever finder = Finder(reader, retriever) ## Evaluate Retriever on its own if eval_retriever_only: retriever_eval_results = retriever.eval() ## Retriever Recall is the proportion of questions for which the correct document containing the answer is ## among the correct documents print("Retriever Recall:", retriever_eval_results["recall"]) ## Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank
def tutorial1_basic_qa_pipeline(): logger = logging.getLogger(__name__) LAUNCH_ELASTICSEARCH = True # ## Document Store # # Haystack finds answers to queries within the documents stored in a `DocumentStore`. The current implementations of # `DocumentStore` include `ElasticsearchDocumentStore`, `FAISSDocumentStore`, `SQLDocumentStore`, and `InMemoryDocumentStore`. # # **Here:** We recommended Elasticsearch as it comes preloaded with features like full-text queries, BM25 retrieval, # and vector storage for text embeddings. # **Alternatives:** If you are unable to setup an Elasticsearch instance, then follow the Tutorial 3 # for using SQL/InMemory document stores. # **Hint**: # This tutorial creates a new document store instance with Wikipedia articles on Game of Thrones. However, you can # configure Haystack to work with your existing document stores. # # Start an Elasticsearch server # You can start Elasticsearch on your local machine instance using Docker. If Docker is not readily available in # your environment (eg., in Colab notebooks), then you can manually download and execute Elasticsearch from source. if LAUNCH_ELASTICSEARCH: logging.info("Starting Elasticsearch ...") status = subprocess.run([ 'docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.9.2' ], shell=True) if status.returncode: raise Exception( "Failed to launch Elasticsearch. If you want to connect to an existing Elasticsearch instance" "then set LAUNCH_ELASTICSEARCH in the script to False.") time.sleep(15) # Connect to Elasticsearch document_store = ElasticsearchDocumentStore(host="localhost", username="", password="", index="document") # ## Preprocessing of documents # # Haystack provides a customizable pipeline for: # - converting files into texts # - cleaning texts # - splitting texts # - writing them to a Document Store # In this tutorial, we download Wikipedia articles about Game of Thrones, apply a basic cleaning function, and add # them in Elasticsearch. # Let's first fetch some documents that we want to query # Here: 517 Wikipedia articles for Game of Thrones doc_dir = "data/article_txt_got" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) # convert files to dicts containing documents that can be indexed to our datastore dicts = convert_files_to_dicts(dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True) # You can optionally supply a cleaning function that is applied to each doc (e.g. to remove footers) # It must take a str as input, and return a str. # Now, let's write the docs to our DB. if LAUNCH_ELASTICSEARCH: document_store.write_documents(dicts) else: logger.warning( "Since we already have a running ES instance we should not index the same documents again. \n" "If you still want to do this call: document_store.write_documents(dicts) manually " ) # ## Initalize Retriever, Reader, & Finder # # ### Retriever # # Retrievers help narrowing down the scope for the Reader to smaller units of text where a given question # could be answered. # # They use some simple but fast algorithm. # **Here:** We use Elasticsearch's default BM25 algorithm # **Alternatives:** # - Customize the `ElasticsearchRetriever`with custom queries (e.g. boosting) and filters # - Use `EmbeddingRetriever` to find candidate documents based on the similarity of # embeddings (e.g. created via Sentence-BERT) # - Use `TfidfRetriever` in combination with a SQL or InMemory Document store for simple prototyping and debugging retriever = ElasticsearchRetriever(document_store=document_store) # Alternative: An in-memory TfidfRetriever based on Pandas dataframes for building quick-prototypes # with SQLite document store. # # from haystack.retriever.tfidf import TfidfRetriever # retriever = TfidfRetriever(document_store=document_store) # ### Reader # # A Reader scans the texts returned by retrievers in detail and extracts the k best answers. They are based # on powerful, but slower deep learning models. # # Haystack currently supports Readers based on the frameworks FARM and Transformers. # With both you can either load a local model or one from Hugging Face's model hub (https://huggingface.co/models). # **Here:** a medium sized RoBERTa QA model using a Reader based on # FARM (https://huggingface.co/deepset/roberta-base-squad2) # **Alternatives (Reader):** TransformersReader (leveraging the `pipeline` of the Transformers package) # **Alternatives (Models):** e.g. "distilbert-base-uncased-distilled-squad" (fast) or # "deepset/bert-large-uncased-whole-word-masking-squad2" (good accuracy) # **Hint:** You can adjust the model to return "no answer possible" with the no_ans_boost. Higher values mean # the model prefers "no answer possible" # # #### FARMReader # Load a local model or any of the QA models on # Hugging Face's model hub (https://huggingface.co/models) reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=True) # #### TransformersReader # Alternative: # reader = TransformersReader( # model_name_or_path="distilbert-base-uncased-distilled-squad", tokenizer="distilbert-base-uncased", use_gpu=-1) # ### Pipeline # # With a Haystack `Pipeline` you can stick together your building blocks to a search pipeline. # Under the hood, `Pipelines` are Directed Acyclic Graphs (DAGs) that you can easily customize for your own use cases. # To speed things up, Haystack also comes with a few predefined Pipelines. One of them is the `ExtractiveQAPipeline` that combines a retriever and a reader to answer our questions. # You can learn more about `Pipelines` in the [docs](https://haystack.deepset.ai/docs/latest/pipelinesmd). from haystack.pipeline import ExtractiveQAPipeline pipe = ExtractiveQAPipeline(reader, retriever) ## Voilà! Ask a question! prediction = pipe.run(query="Who is the father of Arya Stark?", top_k_retriever=10, top_k_reader=5) # prediction = pipe.run(query="Who created the Dothraki vocabulary?", top_k_reader=5) # prediction = pipe.run(query="Who is the sister of Sansa?", top_k_reader=5) print_answers(prediction, details="minimal")
paragraphs = [ x['text'] for x in example['positive_ctxs'] if len(x['text']) < 1200 ] gold_paragraphs.update(paragraphs) return gold_paragraphs train_para = get_all_positive_contexts(StrategyQADataset().train_set()) dev_para = get_all_positive_contexts(StrategyQADataset().dev_set()) all_paras = train_para.union(dev_para) import dpr.experiments.document_store as doc_store_utils elastic_ds = doc_store_utils.get_elastic_document_store() retriever = ElasticsearchRetriever(document_store=elastic_ds) mistakes = 0 for i, s in enumerate(all_paras): retrieve = retriever.retrieve(s, top_k=1)[0].text if not s == retrieve: print('expected ', s) print('got', retrieve) retrieve = [x.text for x in retriever.retrieve(s, top_k=20)] retrieve = [x for x in retrieve if x == s] if retrieve: print('found on second try', retrieve) continue mistakes += 1 print('mistakes', mistakes) print('total', i + 1)
def tutorial5_evaluation(): ############################################## # Settings ############################################## # Choose from Evaluation style from ['retriever_closed', 'reader_closed', 'retriever_reader_open'] # 'retriever_closed' - evaluates only the retriever, based on whether the gold_label document is retrieved. # 'reader_closed' - evaluates only the reader in a closed domain fashion i.e. the reader is given one query # and one document and metrics are calculated on whether the right position in this text is selected by # the model as the answer span (i.e. SQuAD style) # 'retriever_reader_open' - evaluates retriever and reader in open domain fashion i.e. a document is considered # correctly retrieved if it contains the answer string within it. The reader is evaluated based purely on the # predicted string, regardless of which document this came from and the position of the extracted span. style = "retriever_reader_open" # make sure these indices do not collide with existing ones, the indices will be wiped clean before data is inserted doc_index = "tutorial5_docs" label_index = "tutorial5_labels" ############################################## # Code ############################################## launch_es() device, n_gpu = initialize_device_settings(use_cuda=True) # Download evaluation data, which is a subset of Natural Questions development set containing 50 documents doc_dir = "../data/nq" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset_v2.json.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) # Connect to Elasticsearch document_store = ElasticsearchDocumentStore(host="localhost", username="", password="", index="document", create_index=False, embedding_field="emb", embedding_dim=768, excluded_meta_data=["emb"]) # Add evaluation data to Elasticsearch document store # We first delete the custom tutorial indices to not have duplicate elements # and also split our documents into shorter passages using the PreProcessor preprocessor = PreProcessor(split_by="word", split_length=500, split_overlap=0, split_respect_sentence_boundary=False, clean_empty_lines=False, clean_whitespace=False) document_store.delete_all_documents(index=doc_index) document_store.delete_all_documents(index=label_index) document_store.add_eval_data(filename="../data/nq/nq_dev_subset_v2.json", doc_index=doc_index, label_index=label_index, preprocessor=preprocessor) # Let's prepare the labels that we need for the retriever and the reader labels = document_store.get_all_labels_aggregated(index=label_index) # Initialize Retriever retriever = ElasticsearchRetriever(document_store=document_store) # Alternative: Evaluate DensePassageRetriever # Note, that DPR works best when you index short passages < 512 tokens as only those tokens will be used for the embedding. # Here, for nq_dev_subset_v2.json we have avg. num of tokens = 5220(!). # DPR still outperforms Elastic's BM25 by a small margin here. # retriever = DensePassageRetriever(document_store=document_store, # query_embedding_model="facebook/dpr-question_encoder-single-nq-base", # passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base", # use_gpu=True, # embed_title=True, # remove_sep_tok_from_untitled_passages=True) # document_store.update_embeddings(retriever, index=doc_index) # Initialize Reader reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", top_k=4, return_no_answer=True) # Here we initialize the nodes that perform evaluation eval_retriever = EvalDocuments() eval_reader = EvalAnswers( sas_model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2" ) ## Evaluate Retriever on its own in closed domain fashion if style == "retriever_closed": retriever_eval_results = retriever.eval(top_k=10, label_index=label_index, doc_index=doc_index) ## Retriever Recall is the proportion of questions for which the correct document containing the answer is ## among the correct documents print("Retriever Recall:", retriever_eval_results["recall"]) ## Retriever Mean Avg Precision rewards retrievers that give relevant documents a higher rank print("Retriever Mean Avg Precision:", retriever_eval_results["map"]) # Evaluate Reader on its own in closed domain fashion (i.e. SQuAD style) elif style == "reader_closed": reader_eval_results = reader.eval(document_store=document_store, device=device, label_index=label_index, doc_index=doc_index) # Evaluation of Reader can also be done directly on a SQuAD-formatted file without passing the data to Elasticsearch #reader_eval_results = reader.eval_on_file("../data/nq", "nq_dev_subset_v2.json", device=device) ## Reader Top-N-Accuracy is the proportion of predicted answers that match with their corresponding correct answer print("Reader Top-N-Accuracy:", reader_eval_results["top_n_accuracy"]) ## Reader Exact Match is the proportion of questions where the predicted answer is exactly the same as the correct answer print("Reader Exact Match:", reader_eval_results["EM"]) ## Reader F1-Score is the average overlap between the predicted answers and the correct answers print("Reader F1-Score:", reader_eval_results["f1"]) # Evaluate combination of Reader and Retriever in open domain fashion elif style == "retriever_reader_open": # Here is the pipeline definition p = Pipeline() p.add_node(component=retriever, name="ESRetriever", inputs=["Query"]) p.add_node(component=eval_retriever, name="EvalDocuments", inputs=["ESRetriever"]) p.add_node(component=reader, name="QAReader", inputs=["EvalDocuments"]) p.add_node(component=eval_reader, name="EvalAnswers", inputs=["QAReader"]) results = [] for l in labels: res = p.run( query=l.question, top_k_retriever=10, labels=l, top_k_reader=10, index=doc_index, ) results.append(res) eval_retriever.print() print() retriever.print_time() print() eval_reader.print(mode="reader") print() reader.print_time() print() eval_reader.print(mode="pipeline") else: raise ValueError( f'style={style} is not a valid option. Choose from retriever_closed, reader_closed, retriever_reader_open' )
# Connect to Elasticsearch document_store = ElasticsearchDocumentStore(host="localhost", username="", password="", index="document", create_index=False, embedding_field="emb", embedding_dim=768, excluded_meta_data=["emb"]) # Add evaluation data to Elasticsearch database # We first delete the custom tutorial indices to not have duplicate elements document_store.delete_all_documents(index=doc_index) document_store.delete_all_documents(index=label_index) document_store.add_eval_data(filename="../data/nq/nq_dev_subset_v2.json", doc_index=doc_index, label_index=label_index) # Initialize Retriever retriever = ElasticsearchRetriever(document_store=document_store) # Alternative: Evaluate DensePassageRetriever # Note, that DPR works best when you index short passages < 512 tokens as only those tokens will be used for the embedding. # Here, for nq_dev_subset_v2.json we have avg. num of tokens = 5220(!). # DPR still outperforms Elastic's BM25 by a small margin here. # from haystack.retriever.dense import DensePassageRetriever # retriever = DensePassageRetriever(document_store=document_store, embedding_model="dpr-bert-base-nq",batch_size=32) # document_store.update_embeddings(retriever, index="eval_document") # Initialize Reader reader = FARMReader("deepset/roberta-base-squad2") # Initialize Finder which sticks together Reader and Retriever
def test_elasticsearch_retrieval(document_store_with_docs): retriever = ElasticsearchRetriever(document_store=document_store_with_docs) res = retriever.retrieve(query="Who lives in Berlin?") assert res[0].text == "My name is Carla and I live in Berlin" assert len(res) == 3 assert res[0].meta["name"] == "filename1"
def main(): launch_es() document_store = ElasticsearchDocumentStore() es_retriever = ElasticsearchRetriever(document_store=document_store) eval_retriever = EvalRetriever(open_domain=open_domain) reader = FARMReader("deepset/roberta-base-squad2", top_k_per_candidate=4, num_processes=1, return_no_answer=True) eval_reader = EvalReader(debug=True, open_domain=open_domain) # Download evaluation data, which is a subset of Natural Questions development set containing 50 documents doc_dir = "../data/nq" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset_v2.json.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) # Add evaluation data to Elasticsearch document store # We first delete the custom tutorial indices to not have duplicate elements preprocessor = PreProcessor(split_length=500, split_overlap=0, split_respect_sentence_boundary=False, clean_empty_lines=False, clean_whitespace=False) document_store.delete_all_documents(index=doc_index) document_store.delete_all_documents(index=label_index) document_store.add_eval_data(filename="../data/nq/nq_dev_subset_v2.json", doc_index=doc_index, label_index=label_index, preprocessor=preprocessor) labels = document_store.get_all_labels_aggregated(index=label_index) q_to_l_dict = {l.question: {"retriever": l, "reader": l} for l in labels} # Here is the pipeline definition p = Pipeline() p.add_node(component=es_retriever, name="ESRetriever", inputs=["Query"]) p.add_node(component=eval_retriever, name="EvalRetriever", inputs=["ESRetriever"]) p.add_node(component=reader, name="QAReader", inputs=["EvalRetriever"]) p.add_node(component=eval_reader, name="EvalReader", inputs=["QAReader"]) results = [] for i, (q, l) in enumerate(q_to_l_dict.items()): res = p.run( query=q, top_k_retriever=top_k_retriever, labels=l, top_k_reader=10, index=doc_index, # skip_incorrect_retrieval=True ) results.append(res) eval_retriever.print() print() es_retriever.print_time() print() eval_reader.print(mode="reader") print() reader.print_time() print() eval_reader.print(mode="pipeline")
from pprint import pprint from haystack.database.elasticsearch import ElasticsearchDocumentStore from haystack.retriever.sparse import ElasticsearchRetriever if __name__ == '__main__': document_store = ElasticsearchDocumentStore( host="192.168.8.106", username="", password="", index="drqa_wiki", ) retriever = ElasticsearchRetriever(document_store=document_store) while True: q = input("utter question: ") documents = retriever.retrieve(q, top_k=3) pprint([d.text for d in documents])
def test_elasticsearch_retrieval_filters(document_store_with_docs): retriever = ElasticsearchRetriever(document_store=document_store_with_docs) res = retriever.retrieve(query="Who lives in Berlin?", filters={"name": ["filename1"]}) assert res[0].text == "My name is Carla and I live in Berlin" assert len(res) == 1 assert res[0].meta["name"] == "filename1" res = retriever.retrieve(query="Who lives in Berlin?", filters={"name":["filename1"], "meta_field": ["not_existing_value"]}) assert len(res) == 0 res = retriever.retrieve(query="Who lives in Berlin?", filters={"name":["filename1"], "not_existing_field": ["not_existing_value"]}) assert len(res) == 0 retriever = ElasticsearchRetriever(document_store=document_store_with_docs) res = retriever.retrieve(query="Who lives in Berlin?", filters={"name":["filename1"], "meta_field": ["test1","test2"]}) assert res[0].text == "My name is Carla and I live in Berlin" assert len(res) == 1 assert res[0].meta["name"] == "filename1" retriever = ElasticsearchRetriever(document_store=document_store_with_docs) res = retriever.retrieve(query="Who lives in Berlin?", filters={"name":["filename1"], "meta_field":["test2"]}) assert len(res) == 0
def setRetriever(self, documentStore): return ElasticsearchRetriever(document_store=documentStore)
def tutorial14_query_classifier(): #Download and prepare data - 517 Wikipedia articles for Game of Thrones doc_dir = "data/article_txt_got" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) # convert files to dicts containing documents that can be indexed to our datastore got_dicts = convert_files_to_dicts( dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True ) # Initialize DocumentStore and index documents launch_es() document_store = ElasticsearchDocumentStore() document_store.delete_all_documents() document_store.write_documents(got_dicts) # Initialize Sparse retriever es_retriever = ElasticsearchRetriever(document_store=document_store) # Initialize dense retriever dpr_retriever = DensePassageRetriever(document_store) document_store.update_embeddings(dpr_retriever, update_existing_embeddings=False) reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2") # Here we build the pipeline sklearn_keyword_classifier = Pipeline() sklearn_keyword_classifier.add_node(component=SklearnQueryClassifier(), name="QueryClassifier", inputs=["Query"]) sklearn_keyword_classifier.add_node(component=dpr_retriever, name="DPRRetriever", inputs=["QueryClassifier.output_1"]) sklearn_keyword_classifier.add_node(component=es_retriever, name="ESRetriever", inputs=["QueryClassifier.output_2"]) sklearn_keyword_classifier.add_node(component=reader, name="QAReader", inputs=["ESRetriever", "DPRRetriever"]) sklearn_keyword_classifier.draw("pipeline_classifier.png") # Run only the dense retriever on the full sentence query res_1 = sklearn_keyword_classifier.run( query="Who is the father of Arya Stark?", top_k_retriever=10 ) print("DPR Results" + "\n" + "="*15) print_answers(res_1) # Run only the sparse retriever on a keyword based query res_2 = sklearn_keyword_classifier.run( query="arya stark father", top_k_retriever=10 ) print("ES Results" + "\n" + "="*15) print_answers(res_2) # Run only the dense retriever on the full sentence query res_3 = sklearn_keyword_classifier.run( query="which country was jon snow filmed ?", top_k_retriever=10 ) print("DPR Results" + "\n" + "="*15) print_answers(res_3) # Run only the sparse retriever on a keyword based query res_4 = sklearn_keyword_classifier.run( query="jon snow country", top_k_retriever=10 ) print("ES Results" + "\n" + "="*15) print_answers(res_4) # Run only the dense retriever on the full sentence query res_5 = sklearn_keyword_classifier.run( query="who are the younger brothers of arya stark ?", top_k_retriever=10 ) print("DPR Results" + "\n" + "="*15) print_answers(res_5) # Run only the sparse retriever on a keyword based query res_6 = sklearn_keyword_classifier.run( query="arya stark younger brothers", top_k_retriever=10 ) print("ES Results" + "\n" + "="*15) print_answers(res_6) # Here we build the pipeline transformer_keyword_classifier = Pipeline() transformer_keyword_classifier.add_node(component=TransformersQueryClassifier(), name="QueryClassifier", inputs=["Query"]) transformer_keyword_classifier.add_node(component=dpr_retriever, name="DPRRetriever", inputs=["QueryClassifier.output_1"]) transformer_keyword_classifier.add_node(component=es_retriever, name="ESRetriever", inputs=["QueryClassifier.output_2"]) transformer_keyword_classifier.add_node(component=reader, name="QAReader", inputs=["ESRetriever", "DPRRetriever"]) transformer_keyword_classifier.draw("pipeline_classifier.png") # Run only the dense retriever on the full sentence query res_1 = transformer_keyword_classifier.run( query="Who is the father of Arya Stark?", top_k_retriever=10 ) print("DPR Results" + "\n" + "="*15) print_answers(res_1) # Run only the sparse retriever on a keyword based query res_2 = transformer_keyword_classifier.run( query="arya stark father", top_k_retriever=10 ) print("ES Results" + "\n" + "="*15) print_answers(res_2) # Run only the dense retriever on the full sentence query res_3 = transformer_keyword_classifier.run( query="which country was jon snow filmed ?", top_k_retriever=10 ) print("DPR Results" + "\n" + "="*15) print_answers(res_3) # Run only the sparse retriever on a keyword based query res_4 = transformer_keyword_classifier.run( query="jon snow country", top_k_retriever=10 ) print("ES Results" + "\n" + "="*15) print_answers(res_4) # Run only the dense retriever on the full sentence query res_5 = transformer_keyword_classifier.run( query="who are the younger brothers of arya stark ?", top_k_retriever=10 ) print("DPR Results" + "\n" + "="*15) print_answers(res_5) # Run only the sparse retriever on a keyword based query res_6 = transformer_keyword_classifier.run( query="arya stark younger brothers", top_k_retriever=10 ) print("ES Results" + "\n" + "="*15) print_answers(res_6) # Here we build the pipeline transformer_question_classifier = Pipeline() transformer_question_classifier.add_node(component=dpr_retriever, name="DPRRetriever", inputs=["Query"]) transformer_question_classifier.add_node(component=TransformersQueryClassifier(model_name_or_path="shahrukhx01/question-vs-statement-classifier"), name="QueryClassifier", inputs=["DPRRetriever"]) transformer_question_classifier.add_node(component=reader, name="QAReader", inputs=["QueryClassifier.output_1"]) transformer_question_classifier.draw("question_classifier.png") # Run only the QA reader on the question query res_1 = transformer_question_classifier.run( query="Who is the father of Arya Stark?", top_k_retriever=10 ) print("DPR Results" + "\n" + "="*15) print_answers(res_1) # Show only DPR results res_2 = transformer_question_classifier.run( query="Arya Stark was the daughter of a Lord.", top_k_retriever=10 ) print("ES Results" + "\n" + "="*15) res_2 # Here we create the keyword vs question/statement query classifier queries = ["arya stark father","jon snow country", "who is the father of arya stark","which country was jon snow filmed?"] keyword_classifier = TransformersQueryClassifier() for query in queries: result = keyword_classifier.run(query=query) if result[1] == "output_1": category = "question/statement" else: category = "keyword" print(f"Query: {query}, raw_output: {result}, class: {category}") # Here we create the question vs statement query classifier queries = ["Lord Eddard was the father of Arya Stark.","Jon Snow was filmed in United Kingdom.", "who is the father of arya stark?","Which country was jon snow filmed in?"] question_classifier = TransformersQueryClassifier(model_name_or_path="shahrukhx01/question-vs-statement-classifier") for query in queries: result = question_classifier.run(query=query) if result[1] == "output_1": category = "question" else: category = "statement" print(f"Query: {query}, raw_output: {result}, class: {category}")
def get_elastic_search_retriever(document_store): return ElasticsearchRetriever(document_store=document_store)
from fastapi import FastAPI from haystack.document_store.elasticsearch import ElasticsearchDocumentStore from haystack.retriever.sparse import ElasticsearchRetriever from haystack.reader.farm import FARMReader from haystack.pipeline import ExtractiveQAPipeline # initialize doc store, retriever and reader components DOC_STORE = ElasticsearchDocumentStore(host='localhost', username='', password='', index='aurelius') RETRIEVER = ElasticsearchRetriever(DOC_STORE) READER = FARMReader(model_name_or_path='deepset/bert-base-cased-squad2', context_window_size=1500, use_gpu=True) # initialize pipeline PIPELINE = ExtractiveQAPipeline(reader=READER, retriever=RETRIEVER) # initialize API APP = FastAPI() @APP.get('/query') async def get_query(q: str, retriever_limit: int = 10, reader_limit: int = 3): """Makes query to doc store via Haystack pipeline. :param q: Query string representing the question being asked. :type q: str """ # get answers return PIPELINE.run(query=q, top_k_retriever=retriever_limit,
def tutorial11_pipelines(): #Download and prepare data - 517 Wikipedia articles for Game of Thrones doc_dir = "data/article_txt_got" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip" fetch_archive_from_http(url=s3_url, output_dir=doc_dir) # convert files to dicts containing documents that can be indexed to our datastore got_dicts = convert_files_to_dicts(dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True) # Initialize DocumentStore and index documents launch_es() document_store = ElasticsearchDocumentStore() document_store.delete_all_documents() document_store.write_documents(got_dicts) # Initialize Sparse retriever es_retriever = ElasticsearchRetriever(document_store=document_store) # Initialize dense retriever dpr_retriever = DensePassageRetriever(document_store) document_store.update_embeddings(dpr_retriever, update_existing_embeddings=False) reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2") ###################### # Prebuilt Pipelines # ###################### # Extractive QA Pipeline ######################## p_extractive_premade = ExtractiveQAPipeline(reader=reader, retriever=es_retriever) res = p_extractive_premade.run(query="Who is the father of Arya Stark?", top_k_retriever=10, top_k_reader=5) print_answers(res, details="minimal") # Document Search Pipeline ########################## p_retrieval = DocumentSearchPipeline(es_retriever) res = p_retrieval.run(query="Who is the father of Arya Stark?", top_k_retriever=10) print_documents(res, max_text_len=200) # Generator Pipeline ########################## # We set this to True so that the document store returns document embeddings # with each document, this is needed by the Generator document_store.return_embedding = True # Initialize generator rag_generator = RAGenerator() # Generative QA p_generator = GenerativeQAPipeline(generator=rag_generator, retriever=dpr_retriever) res = p_generator.run(query="Who is the father of Arya Stark?", top_k_retriever=10) print_answers(res, details="minimal") # We are setting this to False so that in later pipelines, # we get a cleaner printout document_store.return_embedding = False ############################## # Creating Pipeline Diagrams # ############################## p_extractive_premade.draw("pipeline_extractive_premade.png") p_retrieval.draw("pipeline_retrieval.png") p_generator.draw("pipeline_generator.png") #################### # Custom Pipelines # #################### # Extractive QA Pipeline ######################## # Custom built extractive QA pipeline p_extractive = Pipeline() p_extractive.add_node(component=es_retriever, name="Retriever", inputs=["Query"]) p_extractive.add_node(component=reader, name="Reader", inputs=["Retriever"]) # Now we can run it res = p_extractive.run(query="Who is the father of Arya Stark?", top_k_retriever=10, top_k_reader=5) print_answers(res, details="minimal") p_extractive.draw("pipeline_extractive.png") # Ensembled Retriever Pipeline ############################## # Create ensembled pipeline p_ensemble = Pipeline() p_ensemble.add_node(component=es_retriever, name="ESRetriever", inputs=["Query"]) p_ensemble.add_node(component=dpr_retriever, name="DPRRetriever", inputs=["Query"]) p_ensemble.add_node(component=JoinDocuments(join_mode="concatenate"), name="JoinResults", inputs=["ESRetriever", "DPRRetriever"]) p_ensemble.add_node(component=reader, name="Reader", inputs=["JoinResults"]) p_ensemble.draw("pipeline_ensemble.png") # Run pipeline res = p_ensemble.run( query="Who is the father of Arya Stark?", top_k_retriever=5 #This is top_k per retriever ) print_answers(res, details="minimal") # Query Classification Pipeline ############################### # Decision Nodes help you route your data so that only certain branches of your `Pipeline` are run. # Though this looks very similar to the ensembled pipeline shown above, # the key difference is that only one of the retrievers is run for each request. # By contrast both retrievers are always run in the ensembled approach. class QueryClassifier(): outgoing_edges = 2 def run(self, **kwargs): if "?" in kwargs["query"]: return (kwargs, "output_2") else: return (kwargs, "output_1") # Here we build the pipeline p_classifier = Pipeline() p_classifier.add_node(component=QueryClassifier(), name="QueryClassifier", inputs=["Query"]) p_classifier.add_node(component=es_retriever, name="ESRetriever", inputs=["QueryClassifier.output_1"]) p_classifier.add_node(component=dpr_retriever, name="DPRRetriever", inputs=["QueryClassifier.output_2"]) p_classifier.add_node(component=reader, name="QAReader", inputs=["ESRetriever", "DPRRetriever"]) p_classifier.draw("pipeline_classifier.png") # Run only the dense retriever on the full sentence query res_1 = p_classifier.run(query="Who is the father of Arya Stark?", top_k_retriever=10) print("DPR Results" + "\n" + "=" * 15) print_answers(res_1) # Run only the sparse retriever on a keyword based query res_2 = p_classifier.run(query="Arya Stark father", top_k_retriever=10) print("ES Results" + "\n" + "=" * 15) print_answers(res_2)