def test_final_comma_split_after_number(self): moses = MosesTokenizer() text = u"Sie sollten vor dem Upgrade eine Sicherung dieser Daten erstellen (wie unter Abschnitt 4.1.1, „Sichern aller Daten und Konfigurationsinformationen“ beschrieben). " expected_tokens = [ "Sie", "sollten", "vor", "dem", "Upgrade", "eine", "Sicherung", "dieser", "Daten", "erstellen", "(", "wie", "unter", "Abschnitt", "4.1.1", ",", u"„", "Sichern", "aller", "Daten", "und", "Konfigurationsinformationen", u"“", "beschrieben", ")", ".", ] self.assertEqual(moses.tokenize(text), expected_tokens)
def tokenize_file(iterator, language, processes, quiet): moses = MosesTokenizer(lang=language) moses_tokenize = partial( moses.tokenize, return_str=True, ) return parallel_or_not(iterator, moses_tokenize, processes, quiet)
def tokenize_file( iterator, language, processes, quiet, xml_escape, aggressive_dash_splits, protected_patterns, custom_nb_prefixes, ): moses = MosesTokenizer( lang=language, custom_nonbreaking_prefixes_file=custom_nb_prefixes ) if protected_patterns: if protected_patterns == ":basic:": protected_patterns = moses.BASIC_PROTECTED_PATTERNS elif protected_patterns == ":web:": protected_patterns = moses.WEB_PROTECTED_PATTERNS else: with open(protected_patterns, encoding="utf8") as fin: protected_patterns = [pattern.strip() for pattern in fin.readlines()] moses_tokenize = partial( moses.tokenize, return_str=True, aggressive_dash_splits=aggressive_dash_splits, escape=xml_escape, protected_patterns=protected_patterns, ) return parallel_or_not(iterator, moses_tokenize, processes, quiet)
def test_aggressive_split(self): moses = MosesTokenizer() expected_tokens_wo_aggressive_dash_split = ['foo-bar'] expected_tokens_with_aggressive_dash_split = ['foo', '@-@', 'bar'] assert moses.tokenize('foo-bar') == expected_tokens_wo_aggressive_dash_split assert moses.tokenize('foo-bar', aggressive_dash_splits=True) == expected_tokens_with_aggressive_dash_split
def test_moses_tokenize(self): moses = MosesTokenizer() # Tokenize a sentence. text = u'This, is a sentence with weird\xbb symbols\u2026 appearing everywhere\xbf' expected_tokens = u'This , is a sentence with weird \xbb symbols \u2026 appearing everywhere \xbf' tokenized_text = moses.tokenize(text, return_str=True) assert tokenized_text == expected_tokens # The nonbreaking prefixes should tokenize the final fullstop. assert moses.tokenize('abc def.') == [u'abc', u'def', u'.'] # The nonbreaking prefixes should deal the situation when numeric only prefix is the last token. # In below example, "pp" is the last element, and there is no digit after it. assert moses.tokenize('2016, pp.') == [u'2016', u',', u'pp', u'.'] # Test escape_xml text = "This ain't funny. It's actually hillarious, yet double Ls. | [] < > [ ] & You're gonna shake it off? Don't?" expected_tokens_with_xmlescape = [ 'This', 'ain', ''t', 'funny', '.', 'It', ''s', 'actually', 'hillarious', ',', 'yet', 'double', 'Ls', '.', '|', '[', ']', '<', '>', '[', ']', '&', 'You', ''re', 'gonna', 'shake', 'it', 'off', '?', 'Don', ''t', '?' ] expected_tokens_wo_xmlescape = [ 'This', 'ain', "'t", 'funny', '.', 'It', "'s", 'actually', 'hillarious', ',', 'yet', 'double', 'Ls', '.', '|', '[', ']', '<', '>', '[', ']', '&', 'You', "'re", 'gonna', 'shake', 'it', 'off', '?', 'Don', "'t", '?' ] assert moses.tokenize(text, escape=True) == expected_tokens_with_xmlescape assert moses.tokenize(text, escape=False) == expected_tokens_wo_xmlescape
def tokenize_file(language, processes, xml_escape, aggressive_dash_splits, protected_patterns, custom_nb_prefixes, encoding, quiet): moses = MosesTokenizer(lang=language, custom_nonbreaking_prefixes_file=custom_nb_prefixes) if protected_patterns: with open(protected_patterns, encoding="utf8") as fin: protected_patterns = [ pattern.strip() for pattern in fin.readlines() ] moses_tokenize = partial( moses.tokenize, return_str=True, aggressive_dash_splits=aggressive_dash_splits, escape=xml_escape, protected_patterns=protected_patterns, ) with click.get_text_stream("stdin", encoding=encoding) as fin: with click.get_text_stream("stdout", encoding=encoding) as fout: # If it's single process, joblib parallization is slower, # so just process line by line normally. if processes == 1: for line in tqdm(fin.readlines()): print(moses_tokenize(line), end="\n", file=fout) else: for outline in parallelize_preprocess( moses_tokenize, fin.readlines(), processes, progress_bar=(not quiet)): print(outline, end="\n", file=fout)
def test_protect_patterns(self): moses = MosesTokenizer() text = "this is a webpage https://stackoverflow.com/questions/6181381/how-to-print-variables-in-perl that kicks ass" expected_tokens = [ "this", "is", "a", "webpage", "https://stackoverflow.com/questions/6181381/how-to-print-variables-in-perl", "that", "kicks", "ass", ] assert ( moses.tokenize(text, protected_patterns=moses.BASIC_PROTECTED_PATTERNS) == expected_tokens ) # Testing against pattern from https://github.com/alvations/sacremoses/issues/35 noe_patterns = [ r"(?:http|ftp)s?://" # http:// or https:// r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?))" r"(?::\d+)?" # optional port r"(?:/\w+)*" r"(?:(?:\.[a-z]+)|/?)" ] assert moses.tokenize(text, protected_patterns=noe_patterns) == expected_tokens
def test_opening_brackets(self): tokenizer = MosesTokenizer() detokenizer = MosesDetokenizer() text = "By the mid 1990s a version of the game became a Latvian television series (with a parliamentary setting, and played by Latvian celebrities)." assert detokenizer.detokenize(tokenizer.tokenize(text)) == text
def test_detokenize_with_aggressive_split(self): mt = MosesTokenizer() md = MosesDetokenizer() text = 'foo-bar' assert md.detokenize(mt.tokenize(text, aggressive_dash_splits=True)) == text
def test_dot_splitting(self): moses = MosesTokenizer() text = "The meeting will take place at 11:00 a.m. Tuesday." expected_tokens = ( "The meeting will take place at 11 : 00 a.m. Tuesday .".split() ) self.assertEqual(moses.tokenize(text), expected_tokens)
def tokenize_file(iterator, language, processes, quiet, aggressive_dash_splits): moses = MosesTokenizer(lang=language) moses_tokenize = partial( moses.tokenize, return_str=True, aggressive_dash_splits=aggressive_dash_splits, ) return parallel_or_not(iterator, moses_tokenize, processes, quiet)
def test_opening_brackets(self): moses = MosesTokenizer() text = "By the mid 1990s a version of the game became a Latvian television series (with a parliamentary setting, and played by Latvian celebrities)." # echo By the mid 1990s a version of the game became a Latvian television series (with a parliamentary setting, and played by Latvian celebrities). | perl mosesdecoder\scripts\tokenizer\tokenizer.perl en expected_tokens = "By the mid 1990s a version of the game became a Latvian television series ( with a parliamentary setting , and played by Latvian celebrities ) .".split() assert moses.tokenize(text) == expected_tokens
def test_aggressive_split(self): moses = MosesTokenizer() expected_tokens_wo_aggressive_dash_split = ["foo-bar"] expected_tokens_with_aggressive_dash_split = ["foo", "@-@", "bar"] assert moses.tokenize( "foo-bar") == expected_tokens_wo_aggressive_dash_split assert (moses.tokenize("foo-bar", aggressive_dash_splits=True) == expected_tokens_with_aggressive_dash_split)
def test_stress_has_numeric_only_prefixes(self): """Stress testing to prevent redos.""" moses = MosesTokenizer() for i in range(1, 10): start_time = time.perf_counter() payload = " " + " " * (i * 500) + "" moses.has_numeric_only(payload) stop_time = time.perf_counter() - start_time assert stop_time < 20
def test_final_comma_split_after_number(self): moses = MosesTokenizer() text = u"Sie sollten vor dem Upgrade eine Sicherung dieser Daten erstellen (wie unter Abschnitt 4.1.1, „Sichern aller Daten und Konfigurationsinformationen“ beschrieben). " expected_tokens = [ 'Sie', 'sollten', 'vor', 'dem', 'Upgrade', 'eine', 'Sicherung', 'dieser', 'Daten', 'erstellen', '(', 'wie', 'unter', 'Abschnitt', '4.1.1', ',', u'„', 'Sichern', 'aller', 'Daten', 'und', 'Konfigurationsinformationen', u'“', 'beschrieben', ')', '.' ] self.assertEqual(moses.tokenize(text), expected_tokens)
def test_final_dot_unconditionally(self): # Make sure that it works for examples on # https://github.com/moses-smt/mosesdecoder/pull/204 text = "'So am I." expected_tokens = "'So am I .".split() self.assertEqual(moses.tokenize(text), expected_tokens) moses = MosesTokenizer(lang="fr") text = "Des gens admirent une œuvre d'art." expected_tokens = "Des gens admirent une œuvre d' art .".split() self.assertEqual(moses.tokenize(text), expected_tokens) moses = MosesTokenizer(lang="de") text = "...schwer wie ein iPhone 5." expected_tokens = "... schwer wie ein iPhone 5 .".split() self.assertEqual(moses.tokenize(text), expected_tokens) moses = MosesTokenizer(lang="cz") text = "Dvě děti, které běží bez bot." expected_tokens = "Dvě děti , které běží bez bot .".split() self.assertEqual(moses.tokenize(text), expected_tokens)
def test_mixed_cjk_tokenization(self): tokenizer = MosesTokenizer() detokenizer = MosesDetokenizer() text = u"Japan is 日本 in Japanese." assert tokenizer.tokenize(text) == [ u"Japan", u"is", u"日", u"本", u"in", u"Japanese", u".", ] assert detokenizer.detokenize(tokenizer.tokenize(text)) == text
def test_moses_detokenize(self): mt = MosesTokenizer() md = MosesDetokenizer() text = u'This, is a sentence with weird\xbb symbols\u2026 appearing everywhere\xbf' expected_tokens = mt.tokenize(text) expected_detokens = = u'This , is a sentence with weird \xbb symbols \u2026 appearing everywhere \xbf' assert md.detokenize(expected_tokens) == expected_detokens text = "This ain't funny. It's actually hillarious, yet double Ls. | [] < > [ ] & You're gonna shake it off? Don't?" expected_tokens = [u'This', u'ain', u''t', u'funny', u'.', u'It', u''s', u'actually', u'hillarious', u',', u'yet', u'double', u'Ls', u'.', u'|', u'[', u']', u'<', u'>', u'[', u']', u'&', u'You', u''re', u'gonna', u'shake', u'it', u'off', u'?', u'Don', u''t', u'?'] expected_detokens = "This ain't funny. It's actually hillarious, yet double Ls. | [] < > [] & You're gonna shake it off? Don't?" assert mt.tokenize(text) == expected_tokens assert md.detokenize(expected_tokens) == expected_detokens
def test_moses_detokenize(self): mt = MosesTokenizer() md = MosesDetokenizer() text = ( u"This, is a sentence with weird\xbb symbols\u2026 appearing everywhere\xbf" ) expected_tokens = mt.tokenize(text) expected_detokens = u"This, is a sentence with weird \xbb symbols \u2026 appearing everywhere \xbf" assert md.detokenize(expected_tokens) == expected_detokens text = "This ain't funny. It's actually hillarious, yet double Ls. | [] < > [ ] & You're gonna shake it off? Don't?" expected_tokens = [ u"This", u"ain", u"'t", u"funny", u".", u"It", u"'s", u"actually", u"hillarious", u",", u"yet", u"double", u"Ls", u".", u"|", u"[", u"]", u"<", u">", u"[", u"]", u"&", u"You", u"'re", u"gonna", u"shake", u"it", u"off", u"?", u"Don", u"'t", u"?", ] expected_detokens = "This ain't funny. It's actually hillarious, yet double Ls. | [] < > [] & You're gonna shake it off? Don't?" assert mt.tokenize(text) == expected_tokens assert md.detokenize(expected_tokens) == expected_detokens
def tokenize_file(processes, xml_escape, aggressive_dash_splits): moses = MosesTokenizer() moses_tokenize = partial(moses.tokenize, return_str=True, aggressive_dash_splits=aggressive_dash_splits, escape=xml_escape) with click.get_text_stream('stdin') as fin, click.get_text_stream( 'stdout') as fout: # If it's single process, joblib parallization is slower, # so just process line by line normally. if processes == 1: for line in tqdm(fin.readlines()): print(moses_tokenize(line), end='\n', file=fout) else: for outline in parallelize_preprocess(moses_tokenize, fin.readlines(), processes, progress_bar=True): print(outline, end='\n', file=fout)
def tokenize_file( language, processes, quiet): moses = MosesTokenizer(lang=language) moses_tokenize = partial( moses.tokenize, return_str=True, ) def processor(iterator): if processes == 1: for line in list(iterator): yield moses_tokenize(line) else: for outline in parallelize_preprocess( moses_tokenize, list(iterator), processes, progress_bar=(not quiet) ): yield outline return processor
def test_protect_patterns(self): moses = MosesTokenizer() text = "this is a webpage https://stackoverflow.com/questions/6181381/how-to-print-variables-in-perl that kicks ass" expected_tokens = [ 'this', 'is', 'a', 'webpage', 'https://stackoverflow.com/questions/6181381/how-to-print-variables-in-perl', 'that', 'kicks', 'ass' ] assert moses.tokenize(text, protected_patterns=moses.BASIC_PROTECTED_PATTERNS ) == expected_tokens # Testing against pattern from https://github.com/alvations/sacremoses/issues/35 noe_patterns = [ r'(?:http|ftp)s?://' # http:// or https:// r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?))' r'(?::\d+)?' # optional port r'(?:/\w+)*' r'(?:(?:\.[a-z]+)|/?)' ] assert moses.tokenize( text, protected_patterns=noe_patterns) == expected_tokens
"--output-stdout", action='store_true', help="output as tab-separated similar to the input file instead of 3 files" ) args = parser.parse_args() # opening output files if args.output_stdout == False: args.output_files = [args.output_prefix + '.' + ext for ext in args.exts] fos = [open(fo, 'w') for fo in args.output_files] tokenizer = None if args.tokenize: assert args.source_language, "--source-language must be set if --tokenize flag is used" tokenizer = MosesTokenizer(args.source_language) def cleanup(s, args=None): s = re.sub('<[^>]*>', ' ', s) s = s.strip() if args: if args.remove_begin_hyphens == True and s.startswith('-'): s = s[1:] s = re.sub(' +', ' ', s.strip()) return s for line in tqdm.tqdm(args.input_file): line = line.rstrip()
def test_japanese_tokenization(self): tokenizer = MosesTokenizer(lang="ja") text = u"電話でんわの邪魔じゃまをしないでください" assert tokenizer.tokenize(text) == [text]
def test_korean_tokenization(self): tokenizer = MosesTokenizer(lang="ko") detokenizer = MosesDetokenizer(lang="ko") text = u"세계 에서 가장 강력한." assert tokenizer.tokenize(text) == [u"세계", u"에서", u"가장", u"강력한", u"."] assert detokenizer.detokenize(tokenizer.tokenize(text)) == text
def test_chinese_tokenization(self): tokenizer = MosesTokenizer(lang="zh") text = u"记者 应谦 美国" assert tokenizer.tokenize(text) == [u"记者", u"应谦", u"美国"]
def test_french_apostrophes(self): tokenizer = MosesTokenizer(lang="fr") detokenizer = MosesDetokenizer(lang="fr") text = u"L'amitié nous a fait forts d'esprit" assert detokenizer.detokenize(tokenizer.tokenize(text)) == text
def preprocess(src_file, mt_file, output_dir, tokenize_lang=None): """ pre-process input file before post-editing split at <br> and remove <i> tags and music symbols. store everything in a codes file in output_dir Args: src_file: src_file of the translation to be preprocessed mt_file: output of the mt system file to be preprocessed output_dir: output directory to output the preprocessed files and codes file """ punct_normalizer = MosesPunctNormalizer() # set tokenizer tokenizer = None if tokenize_lang: tokenizer = MosesTokenizer(lang=tokenize_lang) code_file = output_dir+'/codes.'+os.path.basename(mt_file) src_out_file = output_dir+'/'+os.path.basename(src_file)+'.pre' mt_out_file = output_dir+'/'+os.path.basename(mt_file)+'.pre' with open(src_out_file,'w') as fosrc, open(mt_out_file,'w') as fomt, open(code_file,'w') as fcodes, open(src_file) as fsrc, open(mt_file) as fmt: idx=0 for src,mt in zip(fsrc,fmt): src, mt = src.strip(), mt.strip() idx+=1 # standardize br tags src = re.sub('<\s*br\s*\/*>', '<br>', src, flags=re.IGNORECASE) mt = re.sub('<\s*br\s*\/*>', '<br>', mt, flags=re.IGNORECASE) # if number of <br> is same, split and save it as multiple lines src_split = re.split(r'\s*<br>\s*',src) mt_split = re.split(r'\s*<br>\s*',mt) # if the src, mt, do not have the same number of <br>, then do not split it if not (len(src_split) == len(mt_split)): src_split = [src] mt_split = [mt] for src_part, mt_part in zip(src_split, mt_split): code = "{}\t".format(idx) # check if they start with the hyphen has_hyphen = False if src_part.startswith('-'): has_hyphen = True src_part = src_part[1:].lstrip() if mt_part.startswith('-'): has_hyphen = True mt_part = mt_part[1:].lstrip() # check if they start with the music symbol music_syms = ('♫','♬','♪') has_music = False if re.search(r'\s*[{}]\s*'.format(''.join(music_syms)), src_part): has_music = True src_part = re.sub(r'\s*[{}]\s*'.format(''.join(music_syms)), '', src_part) #if mt_part.startswith(music_syms) or mt_part.endswith(music_syms): if re.search(r'\s*[{}]\s*'.format(''.join(music_syms)), mt_part): has_music = True mt_part = re.sub(r'\s*[{}]\s*'.format(''.join(music_syms)), '', mt_part) # check if it has enclosing italics tags. otherwise leave it as it is itag = '<i>' eitag = '</i>' has_itag = False if src_part.startswith(itag) or src_part.endswith(eitag): has_itag = True if mt_part.startswith(itag) or mt_part.endswith(eitag): has_itag = True #if re.match(r'^<i>[^<]*</i>$', src_part): if has_hyphen == True: code += 'HYPHENBEGIN\t' if has_music == True: code += 'MUSIC\t' if has_itag == True: code += 'ITALICTAGS\t' src_part = punct_normalizer.normalize(cleanup(src_part)) mt_part = punct_normalizer.normalize(cleanup(mt_part)) if tokenizer: src_part = " ".join(tokenizer.tokenize(src_part, escape=False)) mt_part = " ".join(tokenizer.tokenize(mt_part, escape=False)) fosrc.write(src_part.strip()+'\n') fomt.write(mt_part.strip()+'\n') fcodes.write("{}\n".format(code))
def test_moses_tokenize(self): moses = MosesTokenizer() # Tokenize a sentence. text = ( u"This, is a sentence with weird\xbb symbols\u2026 appearing everywhere\xbf" ) expected_tokens = u"This , is a sentence with weird \xbb symbols \u2026 appearing everywhere \xbf" tokenized_text = moses.tokenize(text, return_str=True) assert tokenized_text == expected_tokens # The nonbreaking prefixes should tokenize the final fullstop. assert moses.tokenize("abc def.") == [u"abc", u"def", u"."] # The nonbreaking prefixes should deal the situation when numeric only prefix is the last token. # In below example, "pp" is the last element, and there is no digit after it. assert moses.tokenize("2016, pp.") == [u"2016", u",", u"pp", u"."] # Test escape_xml text = "This ain't funny. It's actually hillarious, yet double Ls. | [] < > [ ] & You're gonna shake it off? Don't?" expected_tokens_with_xmlescape = [ "This", "ain", "'t", "funny", ".", "It", "'s", "actually", "hillarious", ",", "yet", "double", "Ls", ".", "|", "[", "]", "<", ">", "[", "]", "&", "You", "'re", "gonna", "shake", "it", "off", "?", "Don", "'t", "?", ] expected_tokens_wo_xmlescape = [ "This", "ain", "'t", "funny", ".", "It", "'s", "actually", "hillarious", ",", "yet", "double", "Ls", ".", "|", "[", "]", "<", ">", "[", "]", "&", "You", "'re", "gonna", "shake", "it", "off", "?", "Don", "'t", "?", ] assert moses.tokenize(text, escape=True) == expected_tokens_with_xmlescape assert moses.tokenize(text, escape=False) == expected_tokens_wo_xmlescape # Test to check https://github.com/alvations/sacremoses/issues/19 text = "this 'is' the thing" expected_tokens = ["this", "'", "is", "'", "the", "thing"] assert moses.tokenize(text, escape=True) == expected_tokens
def test_trailing_dot_apostrophe(self): moses = MosesTokenizer() text = "'Hello.'" expected_tokens = "'Hello . '".split() self.assertEqual(moses.tokenize(text), expected_tokens)