def test_preprocess_word_split():
document = {"text": TEXT}
preprocessor = PreProcessor(split_length=10,
split_overlap=0,
split_by="word",
split_respect_sentence_boundary=False)
documents = preprocessor.process(document)
assert len(documents) == 11
preprocessor = PreProcessor(split_length=15,
split_overlap=0,
split_by="word",
split_respect_sentence_boundary=True)
documents = preprocessor.process(document)
for i, doc in enumerate(documents):
if i == 0:
assert len(doc["text"].split(" ")) == 14
assert len(doc["text"].split(" ")) <= 15 or doc["text"].startswith(
"This is to trick")
assert len(documents) == 8
preprocessor = PreProcessor(split_length=40,
split_overlap=10,
split_by="word",
split_respect_sentence_boundary=True)
documents = preprocessor.process(document)
assert len(documents) == 5
preprocessor = PreProcessor(split_length=5,
split_overlap=0,
split_by="word",
split_respect_sentence_boundary=True)
documents = preprocessor.process(document)
assert len(documents) == 15
def test_preprocess_passage_split():
document = {"text": TEXT}
preprocessor = PreProcessor(split_length=1, split_overlap=0, split_by="passage", split_respect_sentence_boundary=False)
documents = preprocessor.process(document)
assert len(documents) == 3
preprocessor = PreProcessor(split_length=2, split_overlap=0, split_by="passage", split_respect_sentence_boundary=False)
documents = preprocessor.process(document)
assert len(documents) == 2
def test_preprocess_sentence_split():
document = {"text": TEXT}
preprocessor = PreProcessor(split_length=1, split_overlap=0, split_by="sentence")
documents = preprocessor.process(document)
assert len(documents) == 15
preprocessor = PreProcessor(
split_length=10, split_overlap=0, split_by="sentence"
)
documents = preprocessor.process(document)
assert len(documents) == 2
def test_preprocess_passage_split():
document = {"text": TEXT}
preprocessor = PreProcessor(split_length=1,
split_stride=0,
split_by="passage")
documents = preprocessor.process(document)
assert len(documents) == 3
preprocessor = PreProcessor(split_length=2,
split_stride=0,
split_by="passage")
documents = preprocessor.process(document)
assert len(documents) == 2
def loadDocumentsFromFile(self, knowledgeFilePath):
converter = TextConverter(
remove_numeric_tables=False,
valid_languages=["en"])
processor = PreProcessor(
clean_empty_lines=True,
clean_whitespace=True,
clean_header_footer=True,
split_by="passage",
split_length=1,
split_respect_sentence_boundary=False,
split_overlap=0
)
loadedFile = converter.convert(knowledgeFilePath)
documents = processor.process(loadedFile)
for i in range(0, len(documents)):
docMetadata = documents[i]['meta']
docMetadata['name'] = knowledgeFilePath
docMetadata['doucmentID'] = knowledgeFilePath \
+ str(docMetadata['_split_id'])
self.document_store.write_documents(documents)
backagain = self.document_store.get_all_documents()
# for i in range(0,len(backagain)):
# print(i)
# print(":\n")
# print(backagain[i])
# print("---------------")
print("Number of documents loaded", end=": ")
print(self.document_store.get_document_count())
def test_preprocess_word_split():
document = {"text": TEXT}
preprocessor = PreProcessor(split_length=10,
split_stride=0,
split_by="word",
split_respect_sentence_boundary=False)
documents = preprocessor.process(document)
assert len(documents) == 11
preprocessor = PreProcessor(split_length=10,
split_stride=0,
split_by="word",
split_respect_sentence_boundary=True)
documents = preprocessor.process(document)
for doc in documents:
assert len(doc["text"].split(" ")) <= 10 or doc["text"].startswith(
"This is to trick")
assert len(documents) == 15
def test_clean_header_footer():
converter = PDFToTextConverter()
document = converter.convert(file_path=Path("samples/pdf/sample_pdf_2.pdf")) # file contains header/footer
preprocessor = PreProcessor(clean_header_footer=True, split_by=None)
documents = preprocessor.process(document)
assert len(documents) == 1
assert "This is a header." not in documents[0]["text"]
assert "footer" not in documents[0]["text"]
def upload_file_to_document_store(
file: UploadFile = File(...),
remove_numeric_tables: Optional[bool] = Form(REMOVE_NUMERIC_TABLES),
remove_whitespace: Optional[bool] = Form(REMOVE_WHITESPACE),
remove_empty_lines: Optional[bool] = Form(REMOVE_EMPTY_LINES),
remove_header_footer: Optional[bool] = Form(REMOVE_HEADER_FOOTER),
valid_languages: Optional[List[str]] = Form(VALID_LANGUAGES),
split_by: Optional[str] = Form(SPLIT_BY),
split_length: Optional[int] = Form(SPLIT_LENGTH),
split_overlap: Optional[int] = Form(SPLIT_OVERLAP),
split_respect_sentence_boundary: Optional[bool] = Form(
SPLIT_RESPECT_SENTENCE_BOUNDARY),
):
try:
file_path = Path(
FILE_UPLOAD_PATH) / f"{uuid.uuid4().hex}_{file.filename}"
with file_path.open("wb") as buffer:
shutil.copyfileobj(file.file, buffer)
if file.filename.split(".")[-1].lower() == "pdf":
pdf_converter = PDFToTextConverter(
remove_numeric_tables=remove_numeric_tables,
valid_languages=valid_languages)
document = pdf_converter.convert(file_path)
elif file.filename.split(".")[-1].lower() == "txt":
txt_converter = TextConverter(
remove_numeric_tables=remove_numeric_tables,
valid_languages=valid_languages,
)
document = txt_converter.convert(file_path)
else:
raise HTTPException(
status_code=415,
detail=f"Only .pdf and .txt file formats are supported.")
document = {TEXT_FIELD_NAME: document["text"], "name": file.filename}
preprocessor = PreProcessor(
clean_whitespace=remove_whitespace,
clean_header_footer=remove_header_footer,
clean_empty_lines=remove_empty_lines,
split_by=split_by,
split_length=split_length,
split_overlap=split_overlap,
split_respect_sentence_boundary=split_respect_sentence_boundary,
)
documents = preprocessor.process(document)
document_store.write_documents(documents)
return "File upload was successful."
finally:
file.file.close()
def upload_file(
model_id: str = Form(...),
file: UploadFile = File(...),
remove_numeric_tables: Optional[bool] = Form(REMOVE_NUMERIC_TABLES)):
print("uploading file")
if model_id not in MODELS:
raise HTTPException(status_code=400, detail="Invalid model id")
try:
file_path = Path(
FILE_UPLOAD_PATH) / f"{uuid.uuid4().hex}_{file.filename}"
with file_path.open("wb") as buffer:
shutil.copyfileobj(file.file, buffer)
if file.filename.split(".")[-1].lower() == "pdf":
pdf_converter = PDFToTextConverter(
remove_numeric_tables=remove_numeric_tables, )
document = pdf_converter.convert(file_path)
elif file.filename.split(".")[-1].lower() == "txt":
txt_converter = TextConverter(
remove_numeric_tables=remove_numeric_tables)
document = txt_converter.convert(file_path)
else:
raise HTTPException(
status_code=415,
detail=f"Only .pdf and .txt file formats are supported.")
processor = PreProcessor(clean_empty_lines=True,
clean_whitespace=True,
clean_header_footer=True,
split_by="word",
split_length=200,
split_respect_sentence_boundary=True)
docs = processor.process(document)
# Add name field to documents
for doc in docs:
doc['name'] = file.filename
doc_store = MODELS[model_id].finder.retriever.document_store
doc_store.write_documents(docs)
return docs
finally:
file.file.close()
def file_upload(file):
try:
file_path = '/tmp/' + file.name + '_tmp'
with open(file_path, "wb") as buffer:
buffer.write(file.read())
if file.filename.split(".")[-1].lower() == "pdf":
pdf_converter = PDFToTextConverter(
remove_numeric_tables=True, valid_languages=["en"]
)
document = pdf_converter.convert(file_path)
elif file.filename.split(".")[-1].lower() == "txt":
txt_converter = TextConverter(
remove_numeric_tables=True, valid_languages=["en"],
)
document = txt_converter.convert(file_path)
else:
raise HTTPException(status_code=415, detail=f"Only .pdf and .txt file formats are supported.")
document = {TEXT_FIELD_NAME: document["text"], "name": file.filename}
preprocessor = PreProcessor(
clean_whitespace=REMOVE_WHITESPACE,
clean_header_footer=REMOVE_HEADER_FOOTER,
clean_empty_lines=REMOVE_EMPTY_LINES,
split_by=SPLIT_BY,
split_length=SPLIT_LENGTH,
split_respect_sentence_boundary=SPLIT_RESPECT_SENTENCE_BOUNDARY,
)
documents = preprocessor.process(document)
# write the docs to the DB.
document_store.write_documents(documents)
return document_store
finally:
os.remove(file_path)
buffer.close()
def tutorial8_preprocessing():
# This fetches some sample files to work with
doc_dir = "data/preprocessing_tutorial"
s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/preprocessing_tutorial.zip"
fetch_archive_from_http(url=s3_url, output_dir=doc_dir)
"""
## Converters
Haystack's converter classes are designed to help you turn files on your computer into the documents
that can be processed by the Haystack pipeline.
There are file converters for txt, pdf, docx files as well as a converter that is powered by Apache Tika.
"""
# Here are some examples of how you would use file converters
converter = TextConverter(remove_numeric_tables=True,
valid_languages=["en"])
doc_txt = converter.convert(
file_path="data/preprocessing_tutorial/classics.txt", meta=None)
converter = PDFToTextConverter(remove_numeric_tables=True,
valid_languages=["en"])
doc_pdf = converter.convert(
file_path="data/preprocessing_tutorial/bert.pdf", meta=None)
converter = DocxToTextConverter(remove_numeric_tables=True,
valid_languages=["en"])
doc_docx = converter.convert(
file_path="data/preprocessing_tutorial/heavy_metal.docx", meta=None)
# Haystack also has a convenience function that will automatically apply the right converter to each file in a directory.
all_docs = convert_files_to_dicts(dir_path="data/preprocessing_tutorial")
"""
## PreProcessor
The PreProcessor class is designed to help you clean text and split text into sensible units.
File splitting can have a very significant impact on the system's performance.
Have a look at the [Preprocessing](https://haystack.deepset.ai/docs/latest/preprocessingmd)
and [Optimization](https://haystack.deepset.ai/docs/latest/optimizationmd) pages on our website for more details.
"""
# This is a default usage of the PreProcessor.
# Here, it performs cleaning of consecutive whitespaces
# and splits a single large document into smaller documents.
# Each document is up to 1000 words long and document breaks cannot fall in the middle of sentences
# Note how the single document passed into the document gets split into 5 smaller documents
preprocessor = PreProcessor(clean_empty_lines=True,
clean_whitespace=True,
clean_header_footer=False,
split_by="word",
split_length=1000,
split_respect_sentence_boundary=True)
docs_default = preprocessor.process(doc_txt)
print(f"n_docs_input: 1\nn_docs_output: {len(docs_default)}")
"""
## Cleaning
- `clean_empty_lines` will normalize 3 or more consecutive empty lines to be just a two empty lines
- `clean_whitespace` will remove any whitespace at the beginning or end of each line in the text
- `clean_header_footer` will remove any long header or footer texts that are repeated on each page
## Splitting
By default, the PreProcessor will respect sentence boundaries, meaning that documents will not start or end
midway through a sentence.
This will help reduce the possibility of answer phrases being split between two documents.
This feature can be turned off by setting `split_respect_sentence_boundary=False`.
"""
# Not respecting sentence boundary vs respecting sentence boundary
preprocessor_nrsb = PreProcessor(split_respect_sentence_boundary=False)
docs_nrsb = preprocessor_nrsb.process(doc_txt)
print("RESPECTING SENTENCE BOUNDARY")
end_text = docs_default[0]["text"][-50:]
print("End of document: \"..." + end_text + "\"")
print()
print("NOT RESPECTING SENTENCE BOUNDARY")
end_text_nrsb = docs_nrsb[0]["text"][-50:]
print("End of document: \"..." + end_text_nrsb + "\"")
"""
A commonly used strategy to split long documents, especially in the field of Question Answering,
is the sliding window approach. If `split_length=10` and `split_overlap=3`, your documents will look like this:
- doc1 = words[0:10]
- doc2 = words[7:17]
- doc3 = words[14:24]
- ...
You can use this strategy by following the code below.
"""
# Sliding window approach
preprocessor_sliding_window = PreProcessor(
split_overlap=3,
split_length=10,
split_respect_sentence_boundary=False)
docs_sliding_window = preprocessor_sliding_window.process(doc_txt)
doc1 = docs_sliding_window[0]["text"][:200]
doc2 = docs_sliding_window[1]["text"][:100]
doc3 = docs_sliding_window[2]["text"][:100]
print("Document 1: \"" + doc1 + "...\"")
print("Document 2: \"" + doc2 + "...\"")
print("Document 3: \"" + doc3 + "...\"")
#doc_pdf = converter.convert(file_path="data/preprocessing_tutorial/bert.pdf", meta=None)
#converter = DocxToTextConverter(remove_numeric_tables=True, valid_languages=["en"])
#doc_docx = converter.convert(file_path="data/preprocessing_tutorial/heavy_metal.docx", meta=None)
# in our case:
converter = TextConverter(remove_numeric_tables=True, valid_languages=["de"])
doc_txt = converter.convert(file_path="./data/geschichte_19._Jahrhundert.txt",
meta=None)
# TODO: Scraping is not correct yet. E.g. Code civil is incorrect (text after it is left out)
# This is a default usage of the PreProcessor.
# Here, it performs cleaning of consecutive whitespaces
# and splits a single large document into smaller documents.
# Each document is up to 1000 words long and document breaks cannot fall in the middle of sentences
# Note how the single document passed into the document gets split into 5 smaller documents
preprocessor = PreProcessor(clean_empty_lines=True,
clean_whitespace=True,
clean_header_footer=False,
split_by="word",
split_length=100,
split_respect_sentence_boundary=True)
# clean_empty_lines will normalize 3 or more consecutive empty lines to be just a two empty lines
# clean_whitespace will remove any whitespace at the beginning or end of each line in the text
# clean_header_footer will remove any long header or footer texts that are repeated on each page
docs_default = preprocessor.process(doc_txt)
print(f"n_docs_input: 1\nn_docs_output: {len(docs_default)}")
# EOF
def _extract_docs_and_labels_from_dict(document_dict: Dict,
preprocessor: PreProcessor = None,
open_domain: bool = False):
"""Set open_domain to True if you are trying to load open_domain labels (i.e. labels without doc id or start idx)"""
docs = []
labels = []
problematic_ids = []
# get all extra fields from document level (e.g. title)
meta_doc = {
k: v
for k, v in document_dict.items() if k not in ("paragraphs", "title")
}
for paragraph in document_dict["paragraphs"]:
## Create Metadata
cur_meta = {"name": document_dict.get("title", None)}
# all other fields from paragraph level
meta_paragraph = {
k: v
for k, v in paragraph.items() if k not in ("qas", "context")
}
cur_meta.update(meta_paragraph)
# meta from parent document
cur_meta.update(meta_doc)
## Create Document
cur_doc = Document(text=paragraph["context"], meta=cur_meta)
if preprocessor is not None:
splits_dicts = preprocessor.process(cur_doc.to_dict())
# we need to pull in _split_id into the document id for unique reference in labels
# todo: PreProcessor should work on Documents instead of dicts
splits: List[Document] = []
offset = 0
for d in splits_dicts:
id = f"{d['id']}-{d['meta']['_split_id']}"
d["meta"]["_split_offset"] = offset
offset += len(d["text"])
# offset correction based on splitting method
if preprocessor.split_by == "word":
offset += 1
elif preprocessor.split_by == "passage":
offset += 2
else:
raise NotImplementedError
mydoc = Document(text=d["text"], id=id, meta=d["meta"])
splits.append(mydoc)
else:
splits = [cur_doc]
docs.extend(splits)
## Assign Labels to corresponding documents
for qa in paragraph["qas"]:
if not qa.get("is_impossible", False):
for answer in qa["answers"]:
ans = answer["text"]
cur_ans_start = None
# TODO The following block of code means that answer_start is never calculated
# and cur_id is always None for open_domain
# This can be rewritten so that this function could try to calculate offsets
# and populate id in open_domain mode
if open_domain:
cur_ans_start = answer.get("answer_start", 0)
cur_id = '0'
else:
ans_position = cur_doc.text[
answer["answer_start"]:answer["answer_start"] +
len(ans)]
if ans != ans_position:
# do not use answer
problematic_ids.append(qa.get("id", "missing"))
break
# find corresponding document or split
if len(splits) == 1:
cur_id = splits[0].id
cur_ans_start = answer["answer_start"]
else:
for s in splits:
# If answer start offset is contained in passage we assign the label to that passage
if (answer["answer_start"] >=
s.meta["_split_offset"]) and (
answer["answer_start"] <
(s.meta["_split_offset"] +
len(s.text))):
cur_id = s.id
cur_ans_start = answer[
"answer_start"] - s.meta[
"_split_offset"]
# If a document is splitting an answer we add the whole answer text to the document
if s.text[cur_ans_start:cur_ans_start +
len(ans)] != ans:
s.text = s.text[:cur_ans_start] + ans
break
label = Label(
question=qa["question"],
answer=ans,
is_correct_answer=True,
is_correct_document=True,
document_id=cur_id,
offset_start_in_doc=cur_ans_start,
no_answer=qa.get("is_impossible", False),
origin="gold_label",
)
labels.append(label)
else:
# for no_answer we need to assign each split as not fitting to the question
for s in splits:
label = Label(
question=qa["question"],
answer="",
is_correct_answer=True,
is_correct_document=True,
document_id=s.id,
offset_start_in_doc=0,
no_answer=qa.get("is_impossible", False),
origin="gold_label",
)
labels.append(label)
return docs, labels, problematic_ids
def _extract_docs_and_labels_from_dict(document_dict: Dict,
preprocessor: PreProcessor = None):
docs = []
labels = []
# get all extra fields from document level (e.g. title)
meta_doc = {
k: v
for k, v in document_dict.items() if k not in ("paragraphs", "title")
}
for paragraph in document_dict["paragraphs"]:
## Create Metadata
cur_meta = {"name": document_dict.get("title", None)}
# all other fields from paragraph level
meta_paragraph = {
k: v
for k, v in paragraph.items() if k not in ("qas", "context")
}
cur_meta.update(meta_paragraph)
# meta from parent document
cur_meta.update(meta_doc)
## Create Document
cur_doc = Document(text=paragraph["context"], meta=cur_meta)
if preprocessor is not None:
splits_dicts = preprocessor.process(cur_doc.to_dict())
# we need to pull in _split_id into the document id for unique reference in labels
# todo: PreProcessor should work on Documents instead of dicts
splits = []
offset = 0
for d in splits_dicts:
id = f"{d['id']}-{d['meta']['_split_id']}"
d["meta"]["_split_offset"] = offset
offset += len(d["text"])
# offset correction based on splitting method
if preprocessor.split_by == "word":
offset += 1
elif preprocessor.split_by == "passage":
offset += 2
else:
raise NotImplementedError
mydoc = Document(text=d["text"], id=id, meta=d["meta"])
splits.append(mydoc)
else:
splits = [cur_doc]
docs.extend(splits)
## Assign Labels to corresponding documents
for qa in paragraph["qas"]:
if not qa["is_impossible"]:
for answer in qa["answers"]:
ans = answer["text"]
ans_position = cur_doc.text[
answer["answer_start"]:answer["answer_start"] +
len(ans)]
if ans != ans_position:
logger.warning(
f"Answer Text and Answer position mismatch. Skipping Answer"
)
break
# find corresponding document or split
if len(splits) == 1:
cur_id = splits[0].id
cur_ans_start = answer["answer_start"]
else:
for s in splits:
# If answer start offset is contained in passage we assign the label to that passage
if (answer["answer_start"] >=
s.meta["_split_offset"]) and (
answer["answer_start"] <
(s.meta["_split_offset"] +
len(s.text))):
cur_id = s.id
cur_ans_start = answer[
"answer_start"] - s.meta["_split_offset"]
# If a document is splitting an answer we add the whole answer text to the document
if s.text[cur_ans_start:cur_ans_start +
len(ans)] != ans:
s.text = s.text[:cur_ans_start] + ans
break
label = Label(
question=qa["question"],
answer=ans,
is_correct_answer=True,
is_correct_document=True,
document_id=cur_id,
offset_start_in_doc=cur_ans_start,
no_answer=qa["is_impossible"],
origin="gold_label",
)
labels.append(label)
else:
# for no_answer we need to assign each split as not fitting to the question
for s in splits:
label = Label(
question=qa["question"],
answer="",
is_correct_answer=True,
is_correct_document=True,
document_id=s.id,
offset_start_in_doc=0,
no_answer=qa["is_impossible"],
origin="gold_label",
)
labels.append(label)
return docs, labels
and [Optimization](https://haystack.deepset.ai/docs/latest/optimizationmd) pages on our website for more details.
"""
# This is a default usage of the PreProcessor.
# Here, it performs cleaning of consecutive whitespaces
# and splits a single large document into smaller documents.
# Each document is up to 1000 words long and document breaks cannot fall in the middle of sentences
# Note how the single document passed into the document gets split into 5 smaller documents
preprocessor = PreProcessor(clean_empty_lines=True,
clean_whitespace=True,
clean_header_footer=False,
split_by="word",
split_length=1000,
split_respect_sentence_boundary=True)
docs_default = preprocessor.process(doc_txt)
print(f"n_docs_input: 1\nn_docs_output: {len(docs_default)}")
"""
## Cleaning
- `clean_empty_lines` will normalize 3 or more consecutive empty lines to be just a two empty lines
- `clean_whitespace` will remove any whitespace at the beginning or end of each line in the text
- `clean_header_footer` will remove any long header or footer texts that are repeated on each page
## Splitting
By default, the PreProcessor will respect sentence boundaries, meaning that documents will not start or end
midway through a sentence.
This will help reduce the possibility of answer phrases being split between two documents.
This feature can be turned off by setting `split_respect_sentence_boundary=False`.
"""
# Dataframe to dict for haystack
all_dicts = df[['title', 'summary']].rename(columns={
'title': 'name',
'summary': 'text'
}).to_dict(orient='records')
# %%
# clean data
# preprocessing from haystack
preprocessor = PreProcessor(clean_empty_lines=True,
clean_whitespace=True,
clean_header_footer=False,
split_by="word",
split_length=100,
split_respect_sentence_boundary=True,
split_overlap=10)
nested_docs = [preprocessor.process(d) for d in all_dicts]
docs = [d for x in nested_docs for d in x]
# %%
# start FAISS document store and store docs
document_store = FAISSDocumentStore(faiss_index_factory_str="Flat")
document_store.write_documents(docs)
# %%
# initialise storage
from haystack.retriever.dense import DensePassageRetriever
retriever = DensePassageRetriever(
document_store=document_store,
query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
max_seq_len_query=64,