def apply(self, line): tok = hfst.HfstTokenizer() Transducer = hfst.HfstTransducer(self.fallbackTransducer) Transducer.push_weights_to_end() words = hfst.tokenized_fst(tok.tokenize(line)) words.compose(Transducer) words.minimize() return words
def str2fst(str): """ Compile string to FST """ tokenized = tok.tokenize(str) fst = hfst.tokenized_fst(tokenized) return fst
def _compose_block(block, delenv, right_tr, tokenizer): tr = hfst.empty_fst() for word in block: tr.disjunct(hfst.tokenized_fst(tokenizer.tokenize(word))) tr.minimize() tr.compose(delenv) tr.minimize() tr.compose(right_tr) tr.minimize() return tr
def test_tokenized(tok, pathin, pathout, exp, weight=0): tokenized = None if (pathout == None): tokenized = tok.tokenize_one_level(pathin) else: tokenized = tok.tokenize(pathin, pathout) if not hfst.tokenized_fst(tokenized, weight).compare(hfst.regex(exp)): if pathout == None: raise RuntimeError('test_tokenized failed with input: ' + pathin) else: raise RuntimeError('test_tokenized failed with input: ' + pathin + ", " + pathout)
def test_tokenized(tok, pathin, pathout, exp, weight=0): tokenized = None if (pathout == None): tokenized = tok.tokenize_one_level(pathin) else: tokenized = tok.tokenize(pathin, pathout) if not hfst.tokenized_fst(tokenized, weight).compare(hfst.regex(exp)): if pathout == None: raise RuntimeError('test_tokenized failed with input: ' + pathin) else: raise RuntimeError('test_tokenized failed with input: ' + pathin + ", " + pathout)
test_tokenized(tok, 'föbär', 'foofö', '[b:f ä:o r:o]') test_tokenized(tok, 'fööbär', 'föbar', '[fööbär:b 0:a 0:r]') test_tokenized(tok, 'föfööfö', 'föföföföö', '[föö]') tok = hfst.HfstTokenizer() tok.add_skip_symbol('?') tok.add_skip_symbol(' ') test_tokenized(tok, 'How is this tokenized?', None, '[H o w i s t h i s t o k e n i z e d]') tok.add_skip_symbol(' is ') test_tokenized(tok, 'How is this tokenized?', None, '[H o w t h i s t o k e n i z e d]') tok = hfst.HfstTokenizer() tok.add_multichar_symbol(hfst.EPSILON) # TODO: should this be included by default??? test_tokenized(tok, '@_EPSILON_SYMBOL_@foo', None, '[f o o]') if not hfst.tokenized_fst([(hfst.EPSILON,'b'),('f','a'),('o','a'),('o','r')]).compare(hfst.regex('[0:b f:a o:a o:r]')): raise RuntimeError(get_linenumber()) # Is this ok??? if not hfst.regex('"' + hfst.EPSILON + '"').compare(hfst.regex('[0]')): raise RuntimeError(get_linenumber()) if not hfst.regex('"' + hfst.IDENTITY + '"').compare(hfst.regex('[?]')): raise RuntimeError(get_linenumber()) if not hfst.regex('"' + hfst.UNKNOWN + '":"' + hfst.UNKNOWN + '"').compare(hfst.regex('[?:?]')): raise RuntimeError(get_linenumber()) # other python functions if not hfst.empty_fst().compare(hfst.regex('[0-0]')): raise RuntimeError(get_linenumber()) if not hfst.epsilon_fst().compare(hfst.regex('[0]')): raise RuntimeError(get_linenumber())
raise RuntimeError('') if not hfst.fst(('foo', ('bar',5.0))).compare(hfst.regex('{foo}|{bar}::5.0')): raise RuntimeError('') if not hfst.fst(('foo', ('bar',5.0), 'baz', 'Foo', ('Bar',2.4))).compare(hfst.regex('{foo}|{bar}::5.0|{baz}|{Foo}|{Bar}::2.4')): raise RuntimeError('') if not hfst.fst([('foo',-1), ('bar',0), ('baz',3.5)]).compare(hfst.regex('{foo}::-1|{bar}|{baz}::3.5')): raise RuntimeError('') # A dictionary if not hfst.fst({'foo':'foo', 'bar':('foo',1.4), 'baz':(('foo',-1),'BAZ')}).compare(hfst.regex('{foo}|{bar}:{foo}::1.4|{baz}:{foo}::-1|{baz}:{BAZ}')): raise RuntimeError('') # tokenized_fst tok = hfst.HfstTokenizer() tok.add_multichar_symbol('foo') tok.add_multichar_symbol('bar') tr = hfst.tokenized_fst(tok.tokenize('foobar', 'foobaz')) if not tr.compare(hfst.regex('[foo:foo bar:b 0:a 0:z]')): raise RuntimeError('') # HfstBasicTransducer # Create an empty transducer # The transducer has initially one start state (number zero) # that is not final fsm = hfst.HfstBasicTransducer() # Add two states to the transducer fsm.add_state(1) fsm.add_state(2) # Create a transition [foo:bar] leading to state 1 with weight 0.1 tr = hfst.HfstBasicTransition(1, 'foo', 'bar', 0.1) # and add it to state zero fsm.add_transition(0, tr)
test_tokenized(tok, 'föbär', 'foofö', '[b:f ä:o r:o]') test_tokenized(tok, 'fööbär', 'föbar', '[fööbär:b 0:a 0:r]') test_tokenized(tok, 'föfööfö', 'föföföföö', '[föö]') tok = hfst.HfstTokenizer() tok.add_skip_symbol('?') tok.add_skip_symbol(' ') test_tokenized(tok, 'How is this tokenized?', None, '[H o w i s t h i s t o k e n i z e d]') tok.add_skip_symbol(' is ') test_tokenized(tok, 'How is this tokenized?', None, '[H o w t h i s t o k e n i z e d]') tok = hfst.HfstTokenizer() tok.add_multichar_symbol(hfst.EPSILON) # TODO: should this be included by default??? test_tokenized(tok, '@_EPSILON_SYMBOL_@foo', None, '[f o o]') if not hfst.tokenized_fst([(hfst.EPSILON,'b'),('f','a'),('o','a'),('o','r')]).compare(hfst.regex('[0:b f:a o:a o:r]')): raise RuntimeError(get_linenumber()) # Is this ok??? if not hfst.regex('"' + hfst.EPSILON + '"').compare(hfst.regex('[0]')): raise RuntimeError(get_linenumber()) if not hfst.regex('"' + hfst.IDENTITY + '"').compare(hfst.regex('[?]')): raise RuntimeError(get_linenumber()) if not hfst.regex('"' + hfst.UNKNOWN + '":"' + hfst.UNKNOWN + '"').compare(hfst.regex('[?:?]')): raise RuntimeError(get_linenumber()) # other python functions if not hfst.empty_fst().compare(hfst.regex('[0-0]')): raise RuntimeError(get_linenumber()) if not hfst.epsilon_fst().compare(hfst.regex('[0]')): raise RuntimeError(get_linenumber())
raise RuntimeError('') if not hfst.fst(('foo', ('bar',5.0))).compare(hfst.regex('{foo}|{bar}::5.0')): raise RuntimeError('') if not hfst.fst(('foo', ('bar',5.0), 'baz', 'Foo', ('Bar',2.4))).compare(hfst.regex('{foo}|{bar}::5.0|{baz}|{Foo}|{Bar}::2.4')): raise RuntimeError('') if not hfst.fst([('foo',-1), ('bar',0), ('baz',3.5)]).compare(hfst.regex('{foo}::-1|{bar}|{baz}::3.5')): raise RuntimeError('') # A dictionary if not hfst.fst({'foo':'foo', 'bar':('foo',1.4), 'baz':(('foo',-1),'BAZ')}).compare(hfst.regex('{foo}|{bar}:{foo}::1.4|{baz}:{foo}::-1|{baz}:{BAZ}')): raise RuntimeError('') # tokenized_fst tok = hfst.HfstTokenizer() tok.add_multichar_symbol('foo') tok.add_multichar_symbol('bar') tr = hfst.tokenized_fst(tok.tokenize('foobar', 'foobaz')) if not tr.compare(hfst.regex('[foo:foo bar:b 0:a 0:z]')): raise RuntimeError('') # HfstBasicTransducer # Create an empty transducer # The transducer has initially one start state (number zero) # that is not final fsm = hfst.HfstBasicTransducer() # Add two states to the transducer fsm.add_state(1) fsm.add_state(2) # Create a transition [foo:bar] leading to state 1 with weight 0.1 tr = hfst.HfstBasicTransition(1, 'foo', 'bar', 0.1) # and add it to state zero fsm.add_transition(0, tr)
weight = float(line_and_weight[1]) line = line_and_weight[0] tr = None if not pairstrings: input_and_output = line.split(':') if len(input_and_output) == 2: input_and_output[0] = input_and_output[0].strip().rstrip() input_and_output[1] = input_and_output[1].strip().rstrip() if not has_spaces: tr = hfst.fst(input_and_output[0]) tr2 = hfst.fst(input_and_output[1]) tr.cross_product(tr2) else: inputstr = input_and_output[0].split(' ') outputstr = input_and_output[1].split(' ') tr = hfst.tokenized_fst(inputstr) tr2 = hfst.tokenized_fst(outputstr) tr.cross_product(tr2) else: if not has_spaces: tr = hfst.fst(line) else: line = line.split(' ') tr = hfst.tokenized_fst(line) elif has_spaces: line = line.split(' ') symbols = [] for pair in line: symbol_pair = pair.split(':') if len(symbol_pair) == 1: # HFST 4.0: accept also single symbols (identities) symbol_pair.append(symbol_pair[0])