def get_hfst_transducer(name):
hfstpath = os.path.join(DIR, name + '.hfst')
attpath = os.path.join(DIR, name + '.att')
hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE)
hfst.compile_xfst_file(hfstpath, output=sys.stderr)
tr = get_att_transducer(attpath)
tr.lookup_optimize()
return tr
import sys
if len(sys.argv) < 2 or len(sys.argv) > 3:
raise RuntimeError('Usage: prolog2fst.py [sfst|openfst|foma] [[PYTHONPATH]]')
if len(sys.argv) == 3:
sys.path.insert(0, sys.argv[2])
import hfst
if (sys.argv[1] == 'sfst'):
if (hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.SFST_TYPE)):
hfst.set_default_fst_type(hfst.ImplementationType.SFST_TYPE)
else:
raise RuntimeError('Format sfst is not available.')
elif (sys.argv[1] == 'openfst' or sys.argv[1] == "openfst-tropical"):
if (hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)):
hfst.set_default_fst_type(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
else:
raise RuntimeError('Format openfst is not available.')
elif (sys.argv[1] == 'foma'):
if (hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE)):
hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE)
else:
raise RuntimeError('Format foma is not available.')
else:
raise RuntimeError('Transducer format "' + sys.argv[1] + '" not recognized.')
ostr = hfst.HfstOutputStream()
# Works for python 3 only
input_stream=None
if sys.version_info[0] > 2:
import io
import sys
if len(sys.argv) < 3 or len(sys.argv) > 4:
raise RuntimeError(
'Usage: compile_xfst.py [sfst|openfst|foma] FILENAME [[PYTHONPATH]]')
if len(sys.argv) == 4:
sys.path.insert(0, sys.argv[3])
import hfst
if (sys.argv[1] == 'sfst'):
if (hfst.HfstTransducer.is_implementation_type_available(
hfst.ImplementationType.SFST_TYPE)):
hfst.set_default_fst_type(hfst.ImplementationType.SFST_TYPE)
else:
raise RuntimeError('Format sfst is not available.')
elif (sys.argv[1] == 'openfst' or sys.argv[1] == "openfst-tropical"):
if (hfst.HfstTransducer.is_implementation_type_available(
hfst.ImplementationType.TROPICAL_OPENFST_TYPE)):
hfst.set_default_fst_type(
hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
else:
raise RuntimeError('Format openfst is not available.')
elif (sys.argv[1] == 'foma'):
if (hfst.HfstTransducer.is_implementation_type_available(
hfst.ImplementationType.FOMA_TYPE)):
hfst.set_default_fst_type(hfst.ImplementationType.FOMA_TYPE)
else:
raise RuntimeError('Format foma is not available.')
else:
raise RuntimeError('Transducer format "' + sys.argv[1] +
'" not recognized.')
import hfst
import os.path
assert os.path.isfile('streets.txt')
# pmatch transducers are always in ol format, so this has actually no effect...
for type in [hfst.ImplementationType.SFST_TYPE, hfst.ImplementationType.TROPICAL_OPENFST_TYPE, hfst.ImplementationType.FOMA_TYPE]:
if hfst.HfstTransducer.is_implementation_type_available(type):
print(hfst.fst_type_to_string(type))
hfst.set_default_fst_type(type)
# (1) compile the file directly
defs = hfst.compile_pmatch_file('streets.txt')
cont = hfst.PmatchContainer(defs)
assert cont.match("Je marche seul dans l'avenue des Ternes.") == "Je marche seul dans l'<FrenchStreetName>avenue des Ternes</FrenchStreetName>."
# (2) compile the contents of file
with open('streets.txt', 'r') as myfile:
data=myfile.read()
myfile.close()
defs = hfst.compile_pmatch_expression(data)
cont = hfst.PmatchContainer(defs)
assert cont.match("Je marche seul dans l'avenue des Ternes.") == "Je marche seul dans l'<FrenchStreetName>avenue des Ternes</FrenchStreetName>."
# (3) try to compile a nonexistent file
nonexistent_file = 'foofoofoofoofoofoofoofoofoofoofoofoo'
assert not os.path.isfile(nonexistent_file)
try:
hfst.compile_pmatch_file(nonexistent_file)
assert False
except IOError as e:
if int(version[0]) > 2:
def unicode(s, c):
return s
types = []
if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.TROPICAL_OPENFST_TYPE):
types.append(hfst.ImplementationType.TROPICAL_OPENFST_TYPE)
if hfst.HfstTransducer.is_implementation_type_available(hfst.ImplementationType.FOMA_TYPE):
types.append(hfst.ImplementationType.FOMA_TYPE)
print('HERE!!!')
for type in types:
print('\n--- Testing implementation type %s ---\n' % hfst.fst_type_to_string(type))
hfst.set_default_fst_type(type)
tr1 = None
tr2 = None
tr3 = None
type_ = hfst.ImplementationType.TROPICAL_OPENFST_TYPE
ostr = hfst.HfstOutputStream(filename='foobar.hfst', type=type_)
tr_ = hfst.regex('{foo}:{bar}::0.5')
tr_.convert(type_)
ostr.write(tr_)
ostr.write(tr_)
ostr.flush()
ostr.close()
WithFlags=True
elif arg == '-o' or arg == '--output':
ofile = '<next>'
elif arg == '-i' or arg == '--input':
ifile = '<next>'
elif arg == '-f' or arg == '--format':
output_format = '<next>'
elif ofile == '<next>':
ofile = arg
elif ifile == '<next>':
ifile = arg
elif output_format == '<next>':
if arg == 'openfst-tropical' or arg == 'openfst':
output_format = hfst.ImplementationType.TROPICAL_OPENFST_TYPE
elif arg == 'foma':
output_format = hfst.ImplementationType.FOMA_TYPE
elif arg == 'sfst':
output_format = hfst.ImplementationType.SFST_TYPE
else:
raise RuntimeError('Error: hfst-lexc.py: format ' + arg + ' not recognized.')
hfst.set_default_fst_type(output_format)
elif ifile == None:
ifile = arg
elif ofile == None:
ofile = arg
else:
raise RuntimeError('Error: hfst-lexc.py: unknown option: ' + arg)
tr = hfst.compile_lexc_file(ifile, with_flags=WithFlags)
tr.write_to_file(ofile)
import hfst_commandline
import hfst
short_getopts='f:'
long_getopts=['format=',]
impl = hfst.ImplementationType.TROPICAL_OPENFST_TYPE
options = hfst_commandline.hfst_getopt(short_getopts, long_getopts)
for opt in options[0]:
if opt[0] == '-f' or opt[0] == '--format':
impl = hfst_commandline.get_implementation_type(opt[1])
hfst.set_default_fst_type(impl)
result = hfst.compile_sfst_file("")
result.convert(hfst.get_default_fst_type())
ostr = hfst.HfstOutputStream(type=hfst.get_default_fst_type())
ostr.write(result)
inputfilename = argv[i+1]
elif arg == '-o':
skip_next= True
outputfilename = argv[i+1]
elif arg == '-S' or arg == '--has-spaces':
has_spaces=True
elif arg == '-p' or arg == '--pairstrings':
pairstrings=True
elif inputfilename == None:
inputfilename = arg
elif outputfilename == None:
outputfilename = arg
else:
raise RuntimeError('Unknown option: ' + arg)
hfst.set_default_fst_type(impl)
istr = None
if inputfilename != None:
istr = open(inputfilename, 'r')
else:
istr = stdin
ostr = None
if outputfilename != None:
ostr = hfst.HfstOutputStream(filename=outputfilename, type=impl)
else:
ostr = hfst.HfstOutputStream(type=impl)
for line in istr:
line = line.rstrip()
