def cut_sentences_by_rule(text: str, sentence_delimiters: str = "。！？；"):
paragraph = 0
index = 0
buffer = []
results: List[SyntacticUnit] = []
delimiters = set(sentence_delimiters)
for paragraph_text in text.split("\n"):
for char in paragraph_text:
buffer.append(char)
if char in delimiters:
results.append(
SyntacticUnit(
text="".join(buffer),
token=len(results),
index=index,
paragraph=paragraph
)
)
buffer = []
index += 1
if len(buffer) > 0:
results.append(
SyntacticUnit(
text="".join(buffer),
token=len(results),
index=index,
paragraph=paragraph
)
)
buffer = []
elif index == 0:
continue
paragraph += 1
index = 0
return results
def merge_syntactic_units(original_units, filtered_units, tags=None):
units = []
for i in range(len(original_units)):
if filtered_units[i] == '':
continue
text = original_units[i]
token = filtered_units[i]
tag = tags[i][1] if tags else None
sentence = SyntacticUnit(text, token, tag)
sentence.index = i
units.append(sentence)
return units
def merge_syntactic_units(original_units, filtered_units, tags=None):
units = []
for i in range(len(original_units)):
if filtered_units[i] == '':
continue
text = original_units[i]
token = filtered_units[i]
tag = tags[i][1] if tags else None
sentence = SyntacticUnit(text, token, tag)
sentence.index = i
units.append(sentence)
return units
def insert_unit(target_list: List[SyntacticUnit], raw: List[str],
tokens: List[str], pidx: int, sidx: int) -> None:
target_list.append(
SyntacticUnit(text="".join(raw),
token=" ".join(tokens),
index=sidx,
paragraph=pidx))