def _make_transducer(self): segments = self.feature_table.get_segments() transducer = Transducer(segments, name=str(self)) state1 = State('Precede1') state2 = State('Precede2') # After seeing +stress (now it is okay to see +vowel) transducer.add_state(state1) transducer.add_state(state2) transducer.initial_state = state1 transducer.add_final_state(state1) transducer.add_final_state(state2) for segment in segments: segment_symbol = segment.get_symbol() if segment_symbol in yimas_vowels: # segment is vowel transducer.add_arc(Arc(state1, JOKER_SEGMENT, segment, CostVector([1]), state1)) transducer.add_arc(Arc(state2, JOKER_SEGMENT, segment, CostVector([0]), state2)) elif segment_symbol == "'": # segment is stress transducer.add_arc(Arc(state1, JOKER_SEGMENT, segment, CostVector([0]), state2)) transducer.add_arc(Arc(state2, JOKER_SEGMENT, segment, CostVector([0]), state2)) elif segment_symbol in yimas_cons: # segment is consonant transducer.add_arc(Arc(state1, JOKER_SEGMENT, segment, CostVector([0]), state1)) transducer.add_arc(Arc(state2, JOKER_SEGMENT, segment, CostVector([0]), state2)) else: raise ConstraintError("{} not supported in this constraint".format(segment_symbol)) for state in transducer.states: transducer.add_arc(Arc(state, JOKER_SEGMENT, NULL_SEGMENT, CostVector([0]), state)) return transducer
def _make_transducer(self): segments = self.feature_table.get_segments() transducer = Transducer(segments, name=str(self)) state1 = State('Contiguity1') state2 = State('Contiguity2') transducer.add_state(state1) transducer.add_state(state2) transducer.initial_state = state1 transducer.add_final_state(state1) transducer.add_final_state(state2) for segment in segments: transducer.add_arc(Arc(state1, NULL_SEGMENT, segment, CostVector([0]), state1)) transducer.add_arc(Arc(state1, segment, NULL_SEGMENT, CostVector([0]), state1)) transducer.add_arc(Arc(state2, NULL_SEGMENT, segment, CostVector([1]), state1)) transducer.add_arc(Arc(state2, segment, NULL_SEGMENT, CostVector([1]), state1)) segment_symbol = segment.get_symbol() if segment_symbol in yimas_vowels: # segment is vowel transducer.add_arc(Arc(state1, segment, segment, CostVector([0]), state1)) transducer.add_arc(Arc(state2, segment, segment, CostVector([0]), state1)) elif segment_symbol == "'": # segment is stress transducer.add_arc(Arc(state1, segment, segment, CostVector([0]), state2)) transducer.add_arc(Arc(state2, segment, segment, CostVector([0]), state2)) elif segment_symbol in yimas_cons: # segment is consonant transducer.add_arc(Arc(state1, segment, segment, CostVector([0]), state1)) transducer.add_arc(Arc(state2, segment, segment, CostVector([0]), state1)) else: raise ConstraintError("{} not supported in this constraint".format(segment_symbol)) return transducer
def _make_transducer(self): segments = self.feature_table.get_segments() transducer = Transducer(segments, length_of_cost_vectors=0) word_segments = self.get_segments() n = len(self.word_string) states = [State("q{}".format(i), i) for i in range(n+1)] for i, state in enumerate(states): transducer.add_state(state) transducer.add_arc(Arc(state, NULL_SEGMENT, JOKER_SEGMENT, CostVector.get_empty_vector(), state)) if i != n: transducer.add_arc(Arc(states[i], word_segments[i], JOKER_SEGMENT, CostVector.get_empty_vector(), states[i+1])) transducer.initial_state = states[0] transducer.add_final_state(states[n]) return transducer
def _make_transducer(self): segments = self.feature_table.get_segments() transducer = Transducer(segments, length_of_cost_vectors=0) word_segments = self.get_segments() n = len(self.word_string) states = [State("q{}".format(i), i) for i in range(n+1)] for i, state in enumerate(states): transducer.add_state(state) transducer.add_arc(Arc(state, NULL_SEGMENT, JOKER_SEGMENT, CostVector.get_empty_vector(), state)) if i != n: transducer.add_arc(Arc(states[i], word_segments[i], JOKER_SEGMENT, CostVector.get_empty_vector(), states[i+1])) transducer.initial_state = states[0] transducer.add_final_state(states[n]) return transducer
def optimize_transducer_grammar_for_word(word, eval): states_by_index = {} for state in eval.states: if state.index in states_by_index.keys(): states_by_index[state.index].append(state) else: states_by_index[state.index] = [state] arcs_by_index = {} for arc in eval._arcs: if arc.origin_state.index in arcs_by_index.keys(): arcs_by_index[arc.origin_state.index].append(arc) else: arcs_by_index[arc.origin_state.index] = [arc] new_transducer = Transducer(eval.get_alphabet()) state_costs = {} new_transducer.add_state(eval.initial_state) new_transducer.initial_state = eval.initial_state state_costs[eval.initial_state] = CostVector.get_vector(eval.get_length_of_cost_vectors(), 0) for index in range(len(word.get_segments())): new_arcs = _best_arcs(arcs_by_index[index], state_costs) for arc in new_arcs: new_transducer.add_arc(arc) new_transducer.add_state(arc.terminal_state) state_costs[arc.terminal_state] = state_costs[arc.origin_state] + arc.cost_vector new_final_states = [eval.final_states[0]] for state in eval.final_states[1:]: state_cost = state_costs[state] final_cost = state_costs[new_final_states[0]] if state_cost > final_cost: new_final_states = [state] elif state_cost == final_cost: new_final_states.append(state) for state in new_final_states: new_transducer.add_final_state(state) #new_transducer.clear_dead_states(with_impasse_states=True) #TODO give it a try return new_transducer
def test_transducer_clear_dead_states(self): transducer = Transducer(self.feature_table.get_segments()) state1 = State('q1') state2 = State('q2') state3 = State('q3') state4 = State('q4') transducer.add_state(state1) transducer.add_state(state2) transducer.add_state(state3) transducer.add_state(state4) transducer.initial_state = state1 transducer.add_final_state(state2) transducer.add_arc(Arc(state1, JOKER_SEGMENT, NULL_SEGMENT, CostVector([]), state2)) transducer.add_arc(Arc(state1, JOKER_SEGMENT, NULL_SEGMENT, CostVector([]), state1)) transducer.add_arc(Arc(state2, JOKER_SEGMENT, NULL_SEGMENT, CostVector([]), state2)) transducer.add_arc(Arc(state3, JOKER_SEGMENT, NULL_SEGMENT, CostVector([]), state3)) transducer.add_arc(Arc(state4, JOKER_SEGMENT, NULL_SEGMENT, CostVector([]), state3)) transducer.clear_dead_states() self.assertEqual(transducer, get_pickle("clear_dead_states_test_transducer"))
def _make_transducer(self): def compute_num_of_max_satisfied_bundle(segment): i = 0 while i < n and symbol_bundle_characteristic_matrix[segment][i]: i += 1 return i def compute_highest_num_of_satisfied_bundle(segment, j): for k in range(j + 1, 0,-1): if symbol_bundle_characteristic_matrix[segment][k-1]: return k else: return 0 n = len(self.feature_bundles) - 1 segments = self.feature_table.get_segments() transducer = Transducer(segments, name=str(self)) symbol_bundle_characteristic_matrix = {segment: [segment.has_feature_bundle(self.feature_bundles[i]) for i in range(n+1)] for segment in segments} states = {i: {j: 0 for j in range(i)} for i in range(n+1)} initial_state = State('q0|0') # here we use a tuple as label. it will change at the end of this function states[0][0] = initial_state transducer.set_as_single_state(initial_state) if not n: for segment in segments: transducer.add_arc(Arc(states[0][0], JOKER_SEGMENT, segment, CostVector([int(symbol_bundle_characteristic_matrix[segment][0])]), states[0][0])) transducer.add_arc(Arc(states[0][0], JOKER_SEGMENT, NULL_SEGMENT, CostVector([0]), states[0][0])) else: for i in range(0, n+1): for j in range(i): state = State('q{0}|{1}'.format(i,j)) states[i][j] = state transducer.add_state(state) max_num_of_satisfied_bundle_by_segment = {segment: compute_num_of_max_satisfied_bundle(segment) for segment in segments} for segment in segments: transducer.add_arc(Arc(states[0][0], JOKER_SEGMENT, segment, CostVector([0]), states[symbol_bundle_characteristic_matrix[segment][0]][0])) for i in range(n+1): for j in range(i): state = states[i][j] transducer.add_final_state(state) if i != n: for segment in segments: if symbol_bundle_characteristic_matrix[segment][i]: new_state_level = i+1 new_state_mem = min([j+1, max_num_of_satisfied_bundle_by_segment[segment]]) else: new_state_level = compute_highest_num_of_satisfied_bundle(segment, j) new_state_mem = min([max_num_of_satisfied_bundle_by_segment[segment], abs(new_state_level - 1)]) new_terminus = states[new_state_level][new_state_mem] transducer.add_arc(Arc(state, JOKER_SEGMENT, segment, CostVector([0]), new_terminus)) else: # i = n for segment in segments: new_state_level = compute_highest_num_of_satisfied_bundle(segment, j) new_state_mem = min([max_num_of_satisfied_bundle_by_segment[segment], abs(new_state_level - 1)]) new_terminus = states[new_state_level][new_state_mem] transducer.add_arc(Arc(state, JOKER_SEGMENT, segment, CostVector([int(symbol_bundle_characteristic_matrix[segment][i])]), new_terminus)) transducer.clear_dead_states() for state in transducer.states: transducer.add_arc(Arc( state, JOKER_SEGMENT, NULL_SEGMENT, CostVector([0]), state)) return transducer
def _make_transducer(self): def compute_num_of_max_satisfied_bundle(segment): i = 0 while i < n and symbol_bundle_characteristic_matrix[segment][i]: i += 1 return i def compute_highest_num_of_satisfied_bundle(segment, j): for k in range(j + 1, 0, -1): if symbol_bundle_characteristic_matrix[segment][k - 1]: return k else: return 0 n = len(self.feature_bundles) - 1 segments = self.feature_table.get_segments() transducer = Transducer(segments, name=str(self)) symbol_bundle_characteristic_matrix = { segment: [ segment.has_feature_bundle(self.feature_bundles[i]) for i in range(n + 1) ] for segment in segments } states = {i: {j: 0 for j in range(i)} for i in range(n + 1)} initial_state = State( 'q0|0' ) # here we use a tuple as label. it will change at the end of this function states[0][0] = initial_state transducer.set_as_single_state(initial_state) if not n: for segment in segments: transducer.add_arc( Arc( states[0][0], JOKER_SEGMENT, segment, CostVector([ int(symbol_bundle_characteristic_matrix[segment] [0]) ]), states[0][0])) transducer.add_arc( Arc(states[0][0], JOKER_SEGMENT, NULL_SEGMENT, CostVector([0]), states[0][0])) else: for i in range(0, n + 1): for j in range(i): state = State('q{0}|{1}'.format(i, j)) states[i][j] = state transducer.add_state(state) max_num_of_satisfied_bundle_by_segment = { segment: compute_num_of_max_satisfied_bundle(segment) for segment in segments } for segment in segments: transducer.add_arc( Arc( states[0][0], JOKER_SEGMENT, segment, CostVector([0]), states[symbol_bundle_characteristic_matrix[segment] [0]][0])) for i in range(n + 1): for j in range(i): state = states[i][j] transducer.add_final_state(state) if i != n: for segment in segments: if symbol_bundle_characteristic_matrix[segment][i]: new_state_level = i + 1 new_state_mem = min([ j + 1, max_num_of_satisfied_bundle_by_segment[ segment] ]) else: new_state_level = compute_highest_num_of_satisfied_bundle( segment, j) new_state_mem = min([ max_num_of_satisfied_bundle_by_segment[ segment], abs(new_state_level - 1) ]) new_terminus = states[new_state_level][ new_state_mem] transducer.add_arc( Arc(state, JOKER_SEGMENT, segment, CostVector([0]), new_terminus)) transducer.add_arc( Arc(new_terminus, JOKER_SEGMENT, segment, CostVector([0]), new_terminus)) else: # i = n for segment in segments: new_state_level = compute_highest_num_of_satisfied_bundle( segment, j) new_state_mem = min([ max_num_of_satisfied_bundle_by_segment[ segment], abs(new_state_level - 1) ]) new_terminus = states[new_state_level][ new_state_mem] transducer.add_arc( Arc( state, JOKER_SEGMENT, segment, CostVector([ int(symbol_bundle_characteristic_matrix[ segment][i]) ]), new_terminus)) transducer.clear_dead_states() for state in transducer.states: transducer.add_arc( Arc(state, JOKER_SEGMENT, NULL_SEGMENT, CostVector([0]), state)) return transducer
class TestTransducer(unittest.TestCase): def setUp(self): self.feature_table = FeatureTable.load(get_feature_table_fixture("feature_table.json")) self.phonotactic_test_feature_table = FeatureTable.load(get_feature_table_fixture( "phonotactic_test_feature_table.json")) self.transducer = Transducer(self.feature_table.get_segments()) self.state1 = State('q1') self.state2 = State('q2') self.transducer.add_state(self.state1) self.transducer.add_state(self.state2) self.transducer.initial_state = self.state1 self.transducer.add_final_state(self.state2) self.cost_vector1 = CostVector([3, 1, 0]) self.cost_vector2 = CostVector([2, 0, 0]) self.arc = Arc(self.state1, Segment('a', self.feature_table), Segment('b', self.feature_table), CostVector([0, 1, 0]), self.state2) self.transducer.add_arc(self.arc) self.simple_transducer = self.transducer self.loops_transducer = deepcopy(self.transducer) zero_cost_vector = CostVector([0]) segment_a = Segment('a', self.feature_table) segment_b = Segment('b', self.feature_table) self.loops_transducer.add_arc(Arc(self.state1, JOKER_SEGMENT, segment_a, zero_cost_vector, self.state1)) self.loops_transducer.add_arc(Arc(self.state1, JOKER_SEGMENT, segment_b, zero_cost_vector,self.state1)) self.loops_transducer.add_arc(Arc(self.state2, NULL_SEGMENT, segment_a, zero_cost_vector,self.state2)) self.loops_transducer.add_arc(Arc(self.state2, NULL_SEGMENT, segment_b, zero_cost_vector,self.state2)) phonotactic = PhonotacticConstraint([{'cons': '+'}, {'voice': '+'}, {'labial': '+'}], self.phonotactic_test_feature_table).get_transducer() dep = DepConstraint([{'labial': '-'}], self.phonotactic_test_feature_table).get_transducer() max = MaxConstraint([{'voice': '-'}], self.phonotactic_test_feature_table).get_transducer() self.intersection_test_transducer = Transducer.intersection(phonotactic, dep, max) #Transducer tests: def test_transducer_equality(self): feature_table = FeatureTable.load(get_feature_table_fixture("a_b_and_cons_feature_table.json")) faith = FaithConstraint([],feature_table).get_transducer() phonotactic = PhonotacticConstraint([{'cons': '+'}], feature_table).get_transducer() max = MaxConstraint([{'cons': '+'}], feature_table).get_transducer() transducer1 = Transducer.intersection(faith, phonotactic, max) temp_transducer = Transducer.intersection(phonotactic, max) transducer2 = Transducer.intersection(faith, temp_transducer) self.assertEqual(transducer1, transducer2) #write_to_dot_to_file(transducer1, "transducer1") #write_to_dot_to_file(transducer2, "transducer2") #one with constraint set #create with manual intersection def test_transducer_equality_with_deepcopy(self): phonotactic_transducer = PhonotacticConstraint([{'cons': '+'}, {'voice': '+'}, {'labial': '+'}], self.phonotactic_test_feature_table).get_transducer() phonotactic_transducer_copy = deepcopy(phonotactic_transducer) self.assertEqual(phonotactic_transducer, phonotactic_transducer_copy) def test_transducer_equality_with_pickle(self): phonotactic_transducer = PhonotacticConstraint([{'cons': '+'}, {'voice': '+'}, {'labial': '+'}], self.phonotactic_test_feature_table).get_transducer() pickled_phonotactic_transducer = get_pickle("equality_with_pickle_transducer") phonotactic_transducer == pickled_phonotactic_transducer self.assertEqual(phonotactic_transducer, pickled_phonotactic_transducer) def test_transducer_intersection(self): self.assertEqual(self.intersection_test_transducer, get_pickle("intersection_test_transducer")) def test_transducer_clear_dead_states(self): transducer = Transducer(self.feature_table.get_segments()) state1 = State('q1') state2 = State('q2') state3 = State('q3') state4 = State('q4') transducer.add_state(state1) transducer.add_state(state2) transducer.add_state(state3) transducer.add_state(state4) transducer.initial_state = state1 transducer.add_final_state(state2) transducer.add_arc(Arc(state1, JOKER_SEGMENT, NULL_SEGMENT, CostVector([]), state2)) transducer.add_arc(Arc(state1, JOKER_SEGMENT, NULL_SEGMENT, CostVector([]), state1)) transducer.add_arc(Arc(state2, JOKER_SEGMENT, NULL_SEGMENT, CostVector([]), state2)) transducer.add_arc(Arc(state3, JOKER_SEGMENT, NULL_SEGMENT, CostVector([]), state3)) transducer.add_arc(Arc(state4, JOKER_SEGMENT, NULL_SEGMENT, CostVector([]), state3)) transducer.clear_dead_states() self.assertEqual(transducer, get_pickle("clear_dead_states_test_transducer")) def test_get_arcs_by_origin_state(self): initial_state = self.intersection_test_transducer.initial_state arc_list = self.intersection_test_transducer.get_arcs_by_origin_state(initial_state) pickled_arc_list = get_pickle("get_arcs_by_origin_state_arc_list") self.assertTrue(_are_lists_equal(arc_list, pickled_arc_list)) def test_get_arcs_by_terminal_state(self): initial_state = self.intersection_test_transducer.initial_state arc_list = self.intersection_test_transducer.get_arcs_by_origin_state(initial_state) pickled_arc_list = get_pickle("get_arcs_by_terminal_state_arc_list") self.assertTrue(_are_lists_equal(arc_list, pickled_arc_list)) def test_get_range(self): pass # see TestingParserSuite.test_geneare #State tests: def test_state_str(self): self.assertEqual(str(self.state1), "(q1,0)") def test_states_addition(self): new_state = State.states_addition(self.state1, self.state2) self.assertEqual(str(new_state), "(q1|q2,0)") new_state = State.states_addition(self.state1, self.state2) self.assertEqual(str(new_state), "(q1|q2,0)") #Arcs tests: def test_arc_str(self): self.assertEqual(str(self.arc), "['(q1,0)', 'a', 'b', '[0, 1, 0]', '(q2,0)']") #CostVector tests: def test_costVector_operations(self): self.assertEqual(self.cost_vector1 + self.cost_vector2, CostVector([5, 1, 0])) self.assertEqual(self.cost_vector1 * self.cost_vector2, CostVector([3, 1, 0, 2, 0, 0])) self.assertEqual(self.cost_vector1 - self.cost_vector2, CostVector([1, 1, 0])) def test_costVector_comparison(self): self.assertTrue(CostVector([0, 0, 0, 0, 0]) > CostVector([0, 0, 1, 0, 0])) self.assertFalse(CostVector([1, 0, 1]) > CostVector([0, 2, 0])) self.assertTrue(CostVector([1000, 0, 76]) > CostVector.get_inf_vector()) self.assertFalse(CostVector.get_inf_vector() > CostVector([0, 1, 2])) self.assertFalse(CostVector.get_inf_vector() > CostVector.get_inf_vector()) def test_costVector_get_vector_with_size_n_and_number_m(self): self.assertEqual(CostVector.get_vector(4, 0), CostVector([0, 0, 0, 0])) self.assertEqual(CostVector.get_vector(1, 1), CostVector([1])) self.assertEqual(CostVector.get_vector(0, 0), CostVector([])) self.assertEqual(CostVector.get_empty_vector(), CostVector([])) def test_costVector_str(self): self.assertEqual(str(CostVector([1, 1, 0])), "[1, 1, 0]") def test_costVector_illegal_operation(self): with self.assertRaises(CostVectorOperationError): CostVector([1,1]) + CostVector([1]) def test_costVector_concatenation_with_empty_vector(self): cost_vector3 = CostVector([]) self.assertEqual(self.cost_vector1 * cost_vector3, CostVector([3, 1, 0])) self.assertEqual(cost_vector3 * self.cost_vector1, CostVector([3, 1, 0]))