def symbol_pair_to_fst(insym, outsym): """"Return a FST which accepts one the pair string 'insym:outsym'""" bfst = hfst.HfstBasicTransducer() string_pair_path = ((insym, outsym)) bfsa.disjunct(string_pair_path, 0) fst = hfst.fst(bfst) return (fst)
def symbol_to_fsa(sym): """Return a FSA which accepts the one letter string 'sym' The symbol 'sym' may be e.g. a composed Unicode grapheme, i.e. a string of two or more Unicode characters. """ bfsa = hfst.HfstBasicTransducer() string_pair_path = ((sym, sym)) bfsa.disjunct(string_pair_path, 0) fsa = hfst.fst(bfsa) return (fsa)
def main(): """Invoke a simple CLI analyser.""" argp = ArgumentParser() argp.add_argument('-a', '--analyser', metavar='FSA', required=True, help="Path to FSA analyser") argp.add_argument('-i', '--input', metavar="INFILE", type=open, dest="infile", help="source of analysis data in CONLLU") options = argp.parse_args() analyser = load_analyser(options.analyser) sentence = hfst.epsilon_fst() if not options.infile: options.infile = stdin for line in options.infile: line = line.strip() if not line or line == '': print("@SENTENCE_SEPARATOR@") elif line.startswith('#'): print(line) else: refs = line.strip().split('\t') anals = analyse(analyser, refs[1]) if anals: lattice = hfst.empty_fst() for anal in anals: surf = refs[1] deep = anal[0] weight = anal[1] print(surf, deep) bleh = hfst.fst({surf: deep}) lattice.disjunct(bleh) sentence.concatenate(lattice) else: surf = refs[1] deep = refs[1] + "|NOUN|Case=Nom|Number=Sing|Guess=Yes|nsubj" print(surf, deep) bleh = hfst.fst({surf: deep}) sentence.concatenate(bleh) print("@TOKEN SEPARATOR@") foo = hfst.fst("@TOKEN_SEPARATOR@") sentence.concatenate(foo) exit(0)
def align_two_words(in_word, out_word, aligner_fst, zero, number): w1 = hfst.fst(in_word) w1.insert_freely((zero, zero)) w1.minimize() ###print(w1) w2 = hfst.fst(out_word) w2.insert_freely((zero, zero)) w2.minimize() ###print(w2) w3 = hfst.HfstTransducer(w1) w3.compose(aligner_fst) w3.compose(w2) ###print(w1) w3.n_best(number) w3.minimize() ###print(w3) raw_paths = w3.extract_paths(output='raw') if cfg.verbosity >= 10: print("raw_paths:", raw_paths) return raw_paths
def shuffle_with_zeros(string, target_length): """Return a fsa where zeros are inserted in all possible ways string -- the string to which zero symbols are inserted target_length -- how long the strings after insertions must be Returns a fsa which accepts all the strings with the inserted zeros. All strings have exactly target_length symbols. """ result_fsa = hfst.fst(string) l = len(string) if l < target_length: n = target_length - l n_zeros_fsa = hfst.regex(' '.join(n * 'Ø')) result_fsa.shuffle(n_zeros_fsa) result_fsa.minimize() result_fsa.set_name(string) if cfg.verbosity >= 30: print("shuffle_with_zeros:") print(result_fsa) return result_fsa
def to_fst(self) -> hfst.HfstTransducer: return hfst.fst(self.symstr)
def test_fst(input, result): tr1_ = hfst.fst(input) tr2_ = hfst.regex(result) if not tr1_.compare(tr2_): raise RuntimeError('test_fst failed with input: ' + input)
f.close() # Create automaton: # unweighted test_fst('foobar', '[f o o b a r]') test_fst(['foobar'], '[f o o b a r]') test_fst(['foobar', 'foobaz'], '[f o o b a [r|z]]') # with weights test_fst(('foobar', 0.3), '[f o o b a r]::0.3') test_fst([('foobar', 0.5)], '[f o o b a r]::0.5') test_fst(['foobar', ('foobaz', -2)], '[ f o o b a [r|[z::-2]] ]') # Special inputs test_fst('*** FOO ***', '{*** FOO ***}') foo = hfst.fst('') eps = hfst.epsilon_fst() assert(foo.compare(eps)) #try: # foo = hfst.fst('') # raise RuntimeError(get_linenumber()) #except RuntimeError as e: # if not e.__str__() == 'Empty word.': # raise RuntimeError(get_linenumber()) # Create transducer: # unweighted test_fst({'foobar':'foobaz'}, '[f o o b a r:z]') test_fst({'foobar':['foobar','foobaz']}, '[f o o b a [r|r:z]]') test_fst({'foobar':('foobar','foobaz')}, '[f o o b a [r|r:z]]') test_fst({'foobar':'foobaz', 'FOOBAR':('foobar','FOOBAR'), 'Foobar':['Foo','bar','Foobar']}, '[f o o b a r:z] | [F O O B A R] | [F:f O:o O:o B:b A:a R:r] | [F o o b:0 a:0 r:0] | [F:b o:a o:r b:0 a:0 r:0] | [F o o b a r]')
# TransducerTypeMismatchException: if hfst.ImplementationType.FOMA_TYPE in types: hfst.set_default_fst_type(hfst.ImplementationType.TROPICAL_OPENFST_TYPE) tr1 = hfst.regex('foo') tr2 = hfst.regex('bar') tr2.convert(hfst.ImplementationType.FOMA_TYPE) try: tr1.disjunct(tr2) except hfst.exceptions.TransducerTypeMismatchException: print('The implementation types of transducers must be the same.') hfst.set_default_fst_type(type) # fst # One unweighted identity path: if not hfst.fst('foo').compare(hfst.regex('{foo}')): raise RuntimeError('') # Weighted path: a tuple of string and number, e.g. if not hfst.fst(('foo',1.4)).compare(hfst.regex('{foo}::1.4')): raise RuntimeError('') if not hfst.fst(('bar',-3)).compare(hfst.regex('{bar}::-3')): raise RuntimeError('') if not hfst.fst(('baz',0)).compare(hfst.regex('{baz}')): raise RuntimeError('') # Several paths: a list or a tuple of paths and/or weighted paths, e.g. if not hfst.fst(['foo', 'bar']).compare(hfst.regex('{foo}|{bar}')): raise RuntimeError('') if not hfst.fst(('foo', ('bar',5.0))).compare(hfst.regex('{foo}|{bar}::5.0')): raise RuntimeError('') if not hfst.fst(('foo', ('bar',5.0), 'baz', 'Foo', ('Bar',2.4))).compare(hfst.regex('{foo}|{bar}::5.0|{baz}|{Foo}|{Bar}::2.4')): raise RuntimeError('')
def test_fst(input, result): tr1_ = hfst.fst(input) tr2_ = hfst.regex(result) if not tr1_.compare(tr2_): raise RuntimeError('test_fst failed with input: ' + input)
f.close() # Create automaton: # unweighted test_fst('foobar', '[f o o b a r]') test_fst(['foobar'], '[f o o b a r]') test_fst(['foobar', 'foobaz'], '[f o o b a [r|z]]') # with weights test_fst(('foobar', 0.3), '[f o o b a r]::0.3') test_fst([('foobar', 0.5)], '[f o o b a r]::0.5') test_fst(['foobar', ('foobaz', -2)], '[ f o o b a [r|[z::-2]] ]') # Special inputs test_fst('*** FOO ***', '{*** FOO ***}') foo = hfst.fst('') eps = hfst.epsilon_fst() assert(foo.compare(eps)) #try: # foo = hfst.fst('') # raise RuntimeError(get_linenumber()) #except RuntimeError as e: # if not e.__str__() == 'Empty word.': # raise RuntimeError(get_linenumber()) # Create transducer: # unweighted test_fst({'foobar':'foobaz'}, '[f o o b a r:z]') test_fst({'foobar':['foobar','foobaz']}, '[f o o b a [r|r:z]]') test_fst({'foobar':('foobar','foobaz')}, '[f o o b a [r|r:z]]') test_fst({'foobar':'foobaz', 'FOOBAR':('foobar','FOOBAR'), 'Foobar':['Foo','bar','Foobar']}, '[f o o b a r:z] | [F O O B A R] | [F:f O:o O:o B:b A:a R:r] | [F o o b:0 a:0 r:0] | [F:b o:a o:r b:0 a:0 r:0] | [F o o b a r]')
# TransducerTypeMismatchException: if hfst.ImplementationType.FOMA_TYPE in types: hfst.set_default_fst_type(hfst.ImplementationType.TROPICAL_OPENFST_TYPE) tr1 = hfst.regex('foo') tr2 = hfst.regex('bar') tr2.convert(hfst.ImplementationType.FOMA_TYPE) try: tr1.disjunct(tr2) except hfst.exceptions.TransducerTypeMismatchException: print('The implementation types of transducers must be the same.') hfst.set_default_fst_type(type) # fst # One unweighted identity path: if not hfst.fst('foo').compare(hfst.regex('{foo}')): raise RuntimeError('') # Weighted path: a tuple of string and number, e.g. if not hfst.fst(('foo',1.4)).compare(hfst.regex('{foo}::1.4')): raise RuntimeError('') if not hfst.fst(('bar',-3)).compare(hfst.regex('{bar}::-3')): raise RuntimeError('') if not hfst.fst(('baz',0)).compare(hfst.regex('{baz}')): raise RuntimeError('') # Several paths: a list or a tuple of paths and/or weighted paths, e.g. if not hfst.fst(['foo', 'bar']).compare(hfst.regex('{foo}|{bar}')): raise RuntimeError('') if not hfst.fst(('foo', ('bar',5.0))).compare(hfst.regex('{foo}|{bar}::5.0')): raise RuntimeError('') if not hfst.fst(('foo', ('bar',5.0), 'baz', 'Foo', ('Bar',2.4))).compare(hfst.regex('{foo}|{bar}::5.0|{baz}|{Foo}|{Bar}::2.4')): raise RuntimeError('')
for line in istr: line = line.rstrip() weight = None line_and_weight = line.split('\t') if len(line_and_weight) == 2: weight = float(line_and_weight[1]) line = line_and_weight[0] tr = None if not pairstrings: input_and_output = line.split(':') if len(input_and_output) == 2: input_and_output[0] = input_and_output[0].strip().rstrip() input_and_output[1] = input_and_output[1].strip().rstrip() if not has_spaces: tr = hfst.fst(input_and_output[0]) tr2 = hfst.fst(input_and_output[1]) tr.cross_product(tr2) else: inputstr = input_and_output[0].split(' ') outputstr = input_and_output[1].split(' ') tr = hfst.tokenized_fst(inputstr) tr2 = hfst.tokenized_fst(outputstr) tr.cross_product(tr2) else: if not has_spaces: tr = hfst.fst(line) else: line = line.split(' ') tr = hfst.tokenized_fst(line) elif has_spaces:
f.close() # Create automaton: # unweighted test_fst('foobar', '[f o o b a r]') test_fst(['foobar'], '[f o o b a r]') test_fst(['foobar', 'foobaz'], '[f o o b a [r|z]]') # with weights test_fst(('foobar', 0.3), '[f o o b a r]::0.3') test_fst([('foobar', 0.5)], '[f o o b a r]::0.5') test_fst(['foobar', ('foobaz', -2)], '[ f o o b a [r|[z::-2]] ]') # Special inputs test_fst('*** FOO ***', '{*** FOO ***}') try: foo = hfst.fst('') raise RuntimeError(get_linenumber()) except RuntimeError as e: if not e.__str__() == 'Empty word.': raise RuntimeError(get_linenumber()) # Create transducer: # unweighted test_fst({'foobar': 'foobaz'}, '[f o o b a r:z]') test_fst({'foobar': ['foobar', 'foobaz']}, '[f o o b a [r|r:z]]') test_fst({'foobar': ('foobar', 'foobaz')}, '[f o o b a [r|r:z]]') test_fst( { 'foobar': 'foobaz', 'FOOBAR': ('foobar', 'FOOBAR'), 'Foobar': ['Foo', 'bar', 'Foobar']
import sys, hfst algfile = hfst.HfstInputStream("chardist.fst") align = algfile.read() for line in sys.stdin: lst = line.strip().split(sep=":") if len(lst) == 2: f1,f2 = lst else: f1,f2 = lst[0],lst[0] w1 = hfst.fst(f1) w1.insert_freely(("Ø","Ø")) w1.minimize() # print(w1) w2 = hfst.fst(f2) w2.insert_freely(("Ø","Ø")) w2.minimize() # print(w2) w3 = hfst.HfstTransducer(w1) w3.compose(align) w3.compose(w2) # print(w1) w3.n_best(1) w3.minimize() paths = w3.extract_paths(output='text') print(paths.strip())