def get_hfst_transducer(name): hfstpath = os.path.join(DIR, name + '.hfst') attpath = os.path.join(DIR, name + '.att') hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE) hfst.compile_xfst_file(hfstpath, output=sys.stderr) tr = get_att_transducer(attpath) tr.lookup_optimize() return tr
import sys if len(sys.argv) < 2 or len(sys.argv) > 3: raise RuntimeError('Usage: prolog2fst.py [sfst|openfst|foma] [[PYTHONPATH]]') if len(sys.argv) == 3: sys.path.insert(0, sys.argv[2]) import hfst if (sys.argv[1] == 'sfst'): if (hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.SFST_TYPE)): hfst.set_default_fst_type(hfst.ImplementationType.SFST_TYPE) else: raise RuntimeError('Format sfst is not available.') elif (sys.argv[1] == 'openfst' or sys.argv[1] == "openfst-tropical"): if (hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)): hfst.set_default_fst_type(hfst.ImplementationType.TROPICAL_OPENFST_TYPE) else: raise RuntimeError('Format openfst is not available.') elif (sys.argv[1] == 'foma'): if (hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE)): hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE) else: raise RuntimeError('Format foma is not available.') else: raise RuntimeError('Transducer format "' + sys.argv[1] + '" not recognized.') ostr = hfst.HfstOutputStream() # Works for python 3 only input_stream=None if sys.version_info[0] > 2: import io
import sys if len(sys.argv) < 3 or len(sys.argv) > 4: raise RuntimeError( 'Usage: compile_xfst.py [sfst|openfst|foma] FILENAME [[PYTHONPATH]]') if len(sys.argv) == 4: sys.path.insert(0, sys.argv[3]) import hfst if (sys.argv[1] == 'sfst'): if (hfst.HfstTransducer.is_implementation_type_available( hfst.ImplementationType.SFST_TYPE)): hfst.set_default_fst_type(hfst.ImplementationType.SFST_TYPE) else: raise RuntimeError('Format sfst is not available.') elif (sys.argv[1] == 'openfst' or sys.argv[1] == "openfst-tropical"): if (hfst.HfstTransducer.is_implementation_type_available( hfst.ImplementationType.TROPICAL_OPENFST_TYPE)): hfst.set_default_fst_type( hfst.ImplementationType.TROPICAL_OPENFST_TYPE) else: raise RuntimeError('Format openfst is not available.') elif (sys.argv[1] == 'foma'): if (hfst.HfstTransducer.is_implementation_type_available( hfst.ImplementationType.FOMA_TYPE)): hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE) else: raise RuntimeError('Format foma is not available.') else: raise RuntimeError('Transducer format "' + sys.argv[1] + '" not recognized.')
import hfst import os.path assert os.path.isfile('streets.txt') # pmatch transducers are always in ol format, so this has actually no effect... for type in [hfst.ImplementationType.SFST_TYPE, hfst.ImplementationType.TROPICAL_OPENFST_TYPE, hfst.ImplementationType.FOMA_TYPE]: if hfst.HfstTransducer.is_implementation_type_available(type): print(hfst.fst_type_to_string(type)) hfst.set_default_fst_type(type) # (1) compile the file directly defs = hfst.compile_pmatch_file('streets.txt') cont = hfst.PmatchContainer(defs) assert cont.match("Je marche seul dans l'avenue des Ternes.") == "Je marche seul dans l'<FrenchStreetName>avenue des Ternes</FrenchStreetName>." # (2) compile the contents of file with open('streets.txt', 'r') as myfile: data=myfile.read() myfile.close() defs = hfst.compile_pmatch_expression(data) cont = hfst.PmatchContainer(defs) assert cont.match("Je marche seul dans l'avenue des Ternes.") == "Je marche seul dans l'<FrenchStreetName>avenue des Ternes</FrenchStreetName>." # (3) try to compile a nonexistent file nonexistent_file = 'foofoofoofoofoofoofoofoofoofoofoofoo' assert not os.path.isfile(nonexistent_file) try: hfst.compile_pmatch_file(nonexistent_file) assert False except IOError as e:
if int(version[0]) > 2: def unicode(s, c): return s types = [] if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE): types.append(hfst.ImplementationType.TROPICAL_OPENFST_TYPE) if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE): types.append(hfst.ImplementationType.FOMA_TYPE) print('HERE!!!') for type in types: print('\n--- Testing implementation type %s ---\n' % hfst.fst_type_to_string(type)) hfst.set_default_fst_type(type) tr1 = None tr2 = None tr3 = None type_ = hfst.ImplementationType.TROPICAL_OPENFST_TYPE ostr = hfst.HfstOutputStream(filename='foobar.hfst', type=type_) tr_ = hfst.regex('{foo}:{bar}::0.5') tr_.convert(type_) ostr.write(tr_) ostr.write(tr_) ostr.flush() ostr.close()
WithFlags=True elif arg == '-o' or arg == '--output': ofile = '<next>' elif arg == '-i' or arg == '--input': ifile = '<next>' elif arg == '-f' or arg == '--format': output_format = '<next>' elif ofile == '<next>': ofile = arg elif ifile == '<next>': ifile = arg elif output_format == '<next>': if arg == 'openfst-tropical' or arg == 'openfst': output_format = hfst.ImplementationType.TROPICAL_OPENFST_TYPE elif arg == 'foma': output_format = hfst.ImplementationType.FOMA_TYPE elif arg == 'sfst': output_format = hfst.ImplementationType.SFST_TYPE else: raise RuntimeError('Error: hfst-lexc.py: format ' + arg + ' not recognized.') hfst.set_default_fst_type(output_format) elif ifile == None: ifile = arg elif ofile == None: ofile = arg else: raise RuntimeError('Error: hfst-lexc.py: unknown option: ' + arg) tr = hfst.compile_lexc_file(ifile, with_flags=WithFlags) tr.write_to_file(ofile)
import hfst_commandline import hfst short_getopts='f:' long_getopts=['format=',] impl = hfst.ImplementationType.TROPICAL_OPENFST_TYPE options = hfst_commandline.hfst_getopt(short_getopts, long_getopts) for opt in options[0]: if opt[0] == '-f' or opt[0] == '--format': impl = hfst_commandline.get_implementation_type(opt[1]) hfst.set_default_fst_type(impl) result = hfst.compile_sfst_file("") result.convert(hfst.get_default_fst_type()) ostr = hfst.HfstOutputStream(type=hfst.get_default_fst_type()) ostr.write(result)
inputfilename = argv[i+1] elif arg == '-o': skip_next= True outputfilename = argv[i+1] elif arg == '-S' or arg == '--has-spaces': has_spaces=True elif arg == '-p' or arg == '--pairstrings': pairstrings=True elif inputfilename == None: inputfilename = arg elif outputfilename == None: outputfilename = arg else: raise RuntimeError('Unknown option: ' + arg) hfst.set_default_fst_type(impl) istr = None if inputfilename != None: istr = open(inputfilename, 'r') else: istr = stdin ostr = None if outputfilename != None: ostr = hfst.HfstOutputStream(filename=outputfilename, type=impl) else: ostr = hfst.HfstOutputStream(type=impl) for line in istr: line = line.rstrip()