TOK = libhfst.HfstTokenizer()
TOK.add_multichar_symbol("<br />")
TOK.add_skip_symbol("<p>")
TOK.add_skip_symbol("</p>")
spv = TOK.tokenize("<p>A<br />paragraph!</p>")
assert spv == (('A','A'), ('<br />','<br />'), ('p','p'), ('a','a'), ('r','r'), ('a','a'), ('g','g'), ('r','r'), ('a','a'), ('p','p'), ('h','h'), ('!','!'))
## Quick Start to HFST
print("Tutorials")
for type in types:
tr1 = libhfst.HfstTransducer("foo", "bar", type)
tr2 = libhfst.HfstTransducer("bar", "baz", type)
tr1.compose(tr2)
tr1.write_in_att_format(libhfst.hfst_stdout());
# Create an HFST basic transducer [a:b] with transition weight 0.3 and final weight 0.5.
t = libhfst.HfstBasicTransducer()
t.add_state(1)
t.add_transition(0, libhfst.HfstBasicTransition(1, "a", "b", 0.3))
t.set_final_weight(1, 0.5)
# Convert to tropical OpenFst format and push weights toward final state.
T = libhfst.HfstTransducer(t, libhfst.TROPICAL_OPENFST_TYPE)
T.push_weights(libhfst.TO_FINAL_STATE)
# Convert back to HFST basic transducer.
tc = libhfst.HfstBasicTransducer(T)
try:
import libhfst
transducers = []
istr = libhfst.HfstInputStream()
while not istr.is_eof():
transducers.append(istr.read())
istr.close()
if not len(transducers) == 3:
raise RuntimeError('Wrong number of transducers read.')
i = 0
for re in ['föö:bär','0','0-0']:
if not transducers[i].compare(libhfst.regex(re)):
raise RuntimeError('Transducers are not equivalent.')
i += 1
if len(transducers) > 0:
f = libhfst.hfst_stdout()
i=0
transducers[i].write_att(f)
i += 1
while i < len(transducers):
f.write('--\n')
transducers[i].write_att(f)
i += 1
TOK.add_multichar_symbol("<br />")
TOK.add_skip_symbol("<p>")
TOK.add_skip_symbol("</p>")
spv = TOK.tokenize("<p>A<br />paragraph!</p>")
assert spv == (('A', 'A'), ('<br />', '<br />'), ('p', 'p'), ('a', 'a'),
('r', 'r'), ('a', 'a'), ('g', 'g'), ('r', 'r'), ('a', 'a'),
('p', 'p'), ('h', 'h'), ('!', '!'))
## Quick Start to HFST
print("Tutorials")
for type in types:
tr1 = libhfst.HfstTransducer("foo", "bar", type)
tr2 = libhfst.HfstTransducer("bar", "baz", type)
tr1.compose(tr2)
tr1.write_in_att_format(libhfst.hfst_stdout())
# Create an HFST basic transducer [a:b] with transition weight 0.3 and final weight 0.5.
t = libhfst.HfstBasicTransducer()
t.add_state(1)
t.add_transition(0, libhfst.HfstBasicTransition(1, "a", "b", 0.3))
t.set_final_weight(1, 0.5)
# Convert to tropical OpenFst format and push weights toward final state.
T = libhfst.HfstTransducer(t, libhfst.TROPICAL_OPENFST_TYPE)
T.push_weights(libhfst.TO_FINAL_STATE)
# Convert back to HFST basic transducer.
tc = libhfst.HfstBasicTransducer(T)
try:
# Rounding might affect the precision.
import libhfst
transducers = []
istr = libhfst.HfstInputStream()
while not istr.is_eof():
transducers.append(istr.read())
istr.close()
if not len(transducers) == 3:
raise RuntimeError('Wrong number of transducers read.')
i = 0
for re in ['föö:bär', '0', '0-0']:
if not transducers[i].compare(libhfst.regex(re)):
raise RuntimeError('Transducers are not equivalent.')
i += 1
if len(transducers) > 0:
f = libhfst.hfst_stdout()
i = 0
transducers[i].write_att(f)
i += 1
while i < len(transducers):
f.write('--\n')
transducers[i].write_att(f)
i += 1
