def test_lookup_composer(self):
enc = self.segmenting_encoder
word_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
enc.segment_composer = LookupComposer(word_vocab=word_vocab,
char_vocab=self.src_reader.vocab,
hidden_dim=self.layer_dim)
enc.transduce(self.inp_emb(0))
def test_add_multiple_segment_composer(self):
enc = self.segmenting_encoder
word_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
enc.segment_composer = SumMultipleComposer(composers=[
LookupComposer(word_vocab=word_vocab,
char_vocab=self.src_reader.vocab,
hidden_dim=self.layer_dim),
CharNGramComposer(word_vocab=word_vocab,
char_vocab=self.src_reader.vocab,
hidden_dim=self.layer_dim)
])
enc.transduce(self.inp_emb(0))
def test_lookup_composer_learn(self):
enc = self.segmenting_encoder
char_vocab = Vocab(i2w=['a', 'b', 'c', 'd'])
enc.segment_composer = LookupComposer(
word_vocab = None,
char_vocab = char_vocab,
hidden_dim = self.layer_dim,
vocab_size = 4
)
event_trigger.set_train(True)
enc.segment_composer.set_word((0, 1, 2)) # abc 0
enc.segment_composer.transduce([])
enc.segment_composer.set_word((0, 2, 1)) # acb 1
enc.segment_composer.transduce([])
enc.segment_composer.set_word((0, 3, 2)) # adc 2
enc.segment_composer.transduce([])
enc.segment_composer.set_word((0, 1, 2)) # abc 0
enc.segment_composer.transduce([])
enc.segment_composer.set_word((1, 3, 2)) # bdc 3
enc.segment_composer.transduce([])
enc.segment_composer.set_word((3, 3, 3)) # ddd 1 -> acb is the oldest
enc.segment_composer.transduce([])
act = dict(enc.segment_composer.lrucache.items())
exp = {'abc': 0, 'ddd': 1, 'adc': 2, 'bdc': 3}
self.assertDictEqual(act, exp)
enc.segment_composer.set_word((0, 2, 1))
enc.segment_composer.transduce([])
enc.segment_composer.set_word((0, 3, 2))
enc.segment_composer.transduce([])
enc.segment_composer.set_word((0, 1, 2)) # abc 0
enc.segment_composer.transduce([])
enc.segment_composer.set_word((1, 3, 2)) # bdc 3
enc.segment_composer.transduce([])
enc.segment_composer.set_word((3, 3, 3))
enc.segment_composer.transduce([])
enc.segment_composer.set_word((0, 3, 1))
enc.segment_composer.transduce([])
event_trigger.set_train(False)
enc.segment_composer.set_word((3, 3, 2))
enc.segment_composer.transduce([])