def test_norm(self, test_input, expected): pred = normalize(test_input, verbose=False) assert pred == expected
def split_text( in_file: str, out_file: str, vocabulary: List[str] = None, language='eng', remove_brackets=True, do_lower_case=True, min_length=0, max_length=100, additional_split_symbols=None, use_nemo_normalization=False, ): """ Breaks down the in_file roughly into sentences. Each sentence will be on a separate line. Written form of the numbers will be converted to its spoken equivalent, OOV punctuation will be removed. Args: in_file: path to original transcript out_file: path to the output file vocabulary: ASR model vocabulary language: text language remove_brackets: Set to True if square [] and curly {} brackets should be removed from text. Text in square/curly brackets often contains inaudible fragments like notes or translations do_lower_case: flag that determines whether to apply lower case to the in_file text min_length: Min number of chars of the text segment for alignment. Short segments will be combined to be at least min_length (not recommended for multi speaker data). max_length: Max number of chars of the text segment for alignment additional_split_symbols: Additional symbols to use for sentence split if eos sentence split resulted in segments longer than --max_length use_nemo_normalization: Set to True to use NeMo normalization tool to convert numbers from written to spoken format. Normalization using num2words will be applied afterwards to make sure there are no numbers present in the text, otherwise they will be replaced with a space and that could deteriorate segmentation results. """ print(f'Splitting text in {in_file} into sentences.') with open(in_file, "r") as f: transcript = f.read() # remove some symbols for better split into sentences transcript = (transcript.replace("\n", " ").replace("\t", " ").replace( "…", "...").replace("\\", " ").replace("--", " -- ").replace(". . .", "...").replace("‘", "’")) # remove extra space transcript = re.sub(r' +', ' ', transcript) transcript = re.sub(r'(\.+)', '. ', transcript) if remove_brackets: transcript = re.sub(r'(\[.*?\])', ' ', transcript) # remove text in curly brackets transcript = re.sub(r'(\{.*?\})', ' ', transcript) lower_case_unicode = '' upper_case_unicode = '' if language == 'ru': lower_case_unicode = '\u0430-\u04FF' upper_case_unicode = '\u0410-\u042F' elif language not in ['ru', 'eng']: print( f'Consider using {language} unicode letters for better sentence split.' ) # remove space in the middle of the lower case abbreviation to avoid splitting into separate sentences matches = re.findall( r'[a-z' + lower_case_unicode + ']\.\s[a-z' + lower_case_unicode + ']\.', transcript) for match in matches: transcript = transcript.replace(match, match.replace('. ', '.')) # find phrases in quotes with_quotes = re.finditer(r'“[A-Za-z ?]+.*?”', transcript) sentences = [] last_idx = 0 for m in with_quotes: match = m.group() match_idx = m.start() if last_idx < match_idx: sentences.append(transcript[last_idx:match_idx]) sentences.append(match) last_idx = m.end() sentences.append(transcript[last_idx:]) sentences = [s.strip() for s in sentences if s.strip()] # Read and split transcript by utterance (roughly, sentences) split_pattern = f"(?<!\w\.\w.)(?<![A-Z{upper_case_unicode}][a-z{lower_case_unicode}]\.)(?<![A-Z{upper_case_unicode}]\.)(?<=\.|\?|\!|\.”|\?”\!”)\s" new_sentences = [] for sent in sentences: new_sentences.extend(regex.split(split_pattern, sent)) sentences = [s.strip() for s in new_sentences if s.strip()] def additional_split(sentences, split_on_symbols, max_length): if len(split_on_symbols) == 0: return sentences split_on_symbols = split_on_symbols.split('|') def _split(sentences, symbol, max_length): result = [] for s in sentences: if len(s) <= max_length: result.append(s) else: result.extend(s.split(symbol)) return result another_sent_split = [] for sent in sentences: split_sent = [sent] for sym in split_on_symbols: split_sent = _split(split_sent, sym, max_length) another_sent_split.extend(split_sent) sentences = [s.strip() for s in another_sent_split if s.strip()] return sentences sentences = additional_split(sentences, additional_split_symbols, max_length) # check to make sure there will be no utterances for segmentation with only OOV symbols no_space_voc = set(vocabulary) no_space_voc.remove(' ') sentences = [ s for s in sentences if len(no_space_voc.intersection(set(s))) > 0 ] if min_length > 0: sentences_comb = [] sentences_comb.append(sentences[0]) # combines short sentence for i in range(1, len(sentences)): if len(sentences_comb[-1]) < min_length or len( sentences[i]) < min_length: sentences_comb[-1] += ' ' + sentences[i].strip() else: sentences_comb.append(sentences[i].strip()) sentences = "\n".join([s.strip() for s in sentences_comb if s.strip()]) else: sentences = "\n".join([s.strip() for s in sentences if s.strip()]) # save split text with original punctuation and case out_dir, out_file_name = os.path.split(out_file) with open(os.path.join(out_dir, out_file_name[:-4] + '_with_punct.txt'), "w") as f: f.write(sentences) # substitute common abbreviations before applying lower case if language == 'ru': for k, v in RU_ABBREVIATIONS.items(): sentences = sentences.replace(k, v) if do_lower_case: sentences = sentences.lower() if language == 'ru': # replace Latin characters with Russian for k, v in LATIN_TO_RU.items(): sentences = sentences.replace(k, v) if language == 'eng' and use_nemo_normalization: print('Using NeMo normalization tool...') sentences = normalize(sentences, verbose=False) # replace numbers with num2words try: p = re.compile("\d+") new_text = '' match_end = 0 for i, m in enumerate(p.finditer(sentences)): match = m.group() match_start = m.start() if i == 0: new_text = sentences[:match_start] else: new_text += sentences[match_end:match_start] match_end = m.end() new_text += sentences[match_start:match_end].replace( match, num2words(match, lang=language)) new_text += sentences[match_end:] sentences = new_text except NotImplementedError: print( f'{language} might be missing in "num2words" package. Add required language to the choices for the' f'--language argument.') raise # remove all OOV symbols all_symbols = set(sentences) symbols_to_remove = ''.join( all_symbols.difference(set(vocabulary + ['\n']))) sentences = sentences.translate(''.maketrans(symbols_to_remove, len(symbols_to_remove) * ' ')) # remove extra space sentences = re.sub(r' +', ' ', sentences) with open(out_file, "w") as f: f.write(sentences)