def to_fst(self) -> hfst.HfstTransducer:
lexc_file = shared.filenames['lexicon-tr'] + '.lex'
tags = set()
for entry in self.items:
for t in entry.tag:
tags.add(t)
with open_to_write(lexc_file) as lexfp:
lexfp.write('Multichar_Symbols ' +
' '.join(self._lexc_escape(s) \
for s in shared.multichar_symbols+list(tags)) + '\n\n')
lexfp.write('LEXICON Root\n')
for entry in self.items:
lexfp.write('\t' + self._lexc_escape(entry.symstr) + ' # ;\n')
transducer = hfst.compile_lexc_file(full_path(lexc_file))
remove_file(lexc_file)
return transducer
raise RuntimeError(get_linenumber())
if not (TR1.compare(tr1)):
raise RuntimeError(get_linenumber())
if not (TR2.compare(tr2)):
raise RuntimeError(get_linenumber())
# Copy constructor
transducer = hfst.HfstTransducer(TR1)
if not (TR1.compare(transducer)):
raise RuntimeError(get_linenumber())
if not (transducer.compare(TR1)):
raise RuntimeError(get_linenumber())
# Read lexc
tr = hfst.compile_lexc_file('test.lexc')
tr.insert_freely(tr1)
tr.minimize()
tr.insert_freely(('A','B'))
tr.minimize()
# Read sfst
tr = hfst.compile_sfst_file('test.sfstpl')
assert(not (tr == None))
# Substitute
tr = hfst.regex('a a:b b;')
tr.substitute('a', 'A', input=True, output=False)
eq = hfst.regex('A:a A:b b;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
WithFlags=True
elif arg == '-o' or arg == '--output':
ofile = '<next>'
elif arg == '-i' or arg == '--input':
ifile = '<next>'
elif arg == '-f' or arg == '--format':
output_format = '<next>'
elif ofile == '<next>':
ofile = arg
elif ifile == '<next>':
ifile = arg
elif output_format == '<next>':
if arg == 'openfst-tropical' or arg == 'openfst':
output_format = hfst.ImplementationType.TROPICAL_OPENFST_TYPE
elif arg == 'foma':
output_format = hfst.ImplementationType.FOMA_TYPE
elif arg == 'sfst':
output_format = hfst.ImplementationType.SFST_TYPE
else:
raise RuntimeError('Error: hfst-lexc.py: format ' + arg + ' not recognized.')
hfst.set_default_fst_type(output_format)
elif ifile == None:
ifile = arg
elif ofile == None:
ofile = arg
else:
raise RuntimeError('Error: hfst-lexc.py: unknown option: ' + arg)
tr = hfst.compile_lexc_file(ifile, with_flags=WithFlags)
tr.write_to_file(ofile)
import hfst
import hfst_commandline
#treat_warnings_as_errors=False
output_format = hfst.ImplementationType.TROPICAL_OPENFST_TYPE
WithFlags = False
shortopts = 'f:Fi:o:'
longopts = ['format=', 'withFlags', 'Werror', 'input=', 'output=']
options = hfst_commandline.hfst_getopt(shortopts, longopts, 1)
for opt in options[0]:
# if opt[0] == '--Werror':
# treat_warnings_as_errors=True
if opt[0] == '-F' or opt[0] == '--withFlags':
WithFlags = True
elif opt[0] == '-f' or opt[0] == '--format':
output_format = hfst_commandline.get_implementation_type(opt[1])
else:
pass
istr = hfst_commandline.get_one_input_text_stream(options)
ostr = hfst_commandline.get_one_output_hfst_stream(options, output_format)
istr[0].close()
ostr[0].close()
hfst.set_default_fst_type(output_format)
tr = hfst.compile_lexc_file(istr[1], with_flags=WithFlags)
tr.write_to_file(ostr[1])
raise RuntimeError(get_linenumber())
if not (TR1.compare(tr1)):
raise RuntimeError(get_linenumber())
if not (TR2.compare(tr2)):
raise RuntimeError(get_linenumber())
# Copy constructor
transducer = hfst.HfstTransducer(TR1)
if not (TR1.compare(transducer)):
raise RuntimeError(get_linenumber())
if not (transducer.compare(TR1)):
raise RuntimeError(get_linenumber())
# Read lexc
tr = hfst.compile_lexc_file('test.lexc')
tr.insert_freely(tr1)
tr.minimize()
tr.insert_freely(('A','B'))
tr.minimize()
# Read sfst
tr = hfst.compile_sfst_file('test.sfstpl')
assert(not (tr == None))
# Substitute
tr = hfst.regex('a a:b b;')
tr.substitute('a', 'A', input=True, output=False)
eq = hfst.regex('A:a A:b b;')
if not (tr.compare(eq)):
raise RuntimeError(get_linenumber())
print('Fetching lexc file...')
import urllib.request
data = urllib.request.urlopen(
'http://hfst.github.io/downloads/finntreebank.lexc')
s = data.read().decode('utf-8')
data.close()
# todo: implement hfst.compile_lexc(lexc_string)
f = open('finntreebank.lexc', 'w')
f.write(s)
f.close()
import hfst
print('Compiling the file...')
tr = hfst.compile_lexc_file('finntreebank.lexc')
assert (tr != None)
print('Inverting the transducer...')
tr.invert()
tr.minimize()
print('Testing the result:')
print('')
for word in ('testi', 'xtesti', 'alusta'):
print(word + ':')
print(tr.lookup(word, output='text'))
# todo: empty result should contain a newline?
# todo: the indentation of weights when there are several results