def fetch_data_from_repo(
doc_dir="resources/data/website_data/",
s3_url="https://github.com/Thabo-5/Chatbot-scraper/raw/master/txt_files.zip",
doc_store=FAISSDocumentStore()):
"""
Function to download data from s3 bucket/ github
Parameters
----------
doc_dir (str): path to destination folder
s3_url (str): path to download zipped data
doc_store (class): Haystack document store
Returns
-------
document_store (object): Haystack document store object
"""
document_store = doc_store
fetch_archive_from_http(url=s3_url, output_dir=doc_dir)
import os
for filename in os.listdir(path=doc_dir):
with open(os.path.join(doc_dir, filename),
'r',
encoding='utf-8',
errors='replace') as file:
text = file.read()
file.close()
with open(os.path.join(doc_dir, filename),
'w',
encoding='utf-8',
errors='replace') as file:
file.write(text)
file.close()
# Convert files to dicts
dicts = convert_files_to_dicts(dir_path=doc_dir,
clean_func=clean_wiki_text,
split_paragraphs=True)
# Now, let's write the dicts containing documents to our DB.
document_store.write_documents(dicts)
return document_store
# You can start Elasticsearch on your local machine instance using Docker. If Docker is not readily available in
# your environment (eg., in Colab notebooks), then you can manually download and execute Elasticsearch from source.
if LAUNCH_ELASTICSEARCH:
logging.info("Starting Elasticsearch ...")
status = subprocess.run(
['docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.6.2'], shell=True
)
if status.returncode:
raise Exception("Failed to launch Elasticsearch. If you want to connect to an existing Elasticsearch instance"
"then set LAUNCH_ELASTICSEARCH in the script to False.")
time.sleep(30)
# Download evaluation data, which is a subset of Natural Questions development set containing 50 documents
doc_dir = "../data/nq"
s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset.json.zip"
fetch_archive_from_http(url=s3_url, output_dir=doc_dir)
# Connect to Elasticsearch
document_store = ElasticsearchDocumentStore(host="localhost", username="", password="", index="document", create_index=False)
# Add evaluation data to Elasticsearch database
if LAUNCH_ELASTICSEARCH:
document_store.add_eval_data("../data/nq/nq_dev_subset.json")
else:
logger.warning("Since we already have a running ES instance we should not index the same documents again."
"If you still want to do this call: 'document_store.add_eval_data('../data/nq/nq_dev_subset.json')' manually ")
# Initialize Retriever
retriever = ElasticsearchRetriever(document_store=document_store)
# Initialize Reader
reader = FARMReader("deepset/roberta-base-squad2")