def testVocabMappingReplace(self):
old = self._saveVocab("old", ["1", "2", "3", "4"])
new = self._saveVocab("new", ["1", "6", "5", "3", "7"])
mapping, new_vocab = checkpoint._get_vocabulary_mapping(old, new, "replace")
self.assertEqual(5 + 1, len(mapping)) # new + <unk>
self.assertAllEqual([0, -1, -1, 2, -1, 4], mapping)
self.assertAllEqual(["1", "6", "5", "3", "7"], new_vocab.words)
def testVocabMappingMerge(self):
old = self._saveVocab("old", ["1", "2", "3", "4"])
new = self._saveVocab("new", ["1", "6", "3", "5", "7"])
mapping, new_vocab = checkpoint._get_vocabulary_mapping(old, new, "merge")
self.assertEqual(4 + 5 - 2 + 1, len(mapping)) # old + new - common + <unk>
self.assertAllEqual([0, 1, 2, 3, -1, -1, -1, 4], mapping)
self.assertAllEqual(["1", "2", "3", "4", "6", "5", "7"], new_vocab.words)
def testVocabMappingMerge(self):
old = self._saveVocab("old", [1, 2, 3, 4])
new = self._saveVocab("new", [1, 6, 3, 5, 7])
mapping = checkpoint._get_vocabulary_mapping(old, new, "merge")
self.assertEqual(4 + 5 - 2 + 1,
len(mapping)) # old + new - common + <unk>
self.assertAllEqual([0, 1, 2, 3, -1, -1, -1, 4], mapping)
def testVocabMappingReplace(self):
old = self._saveVocab("old", [1, 2, 3, 4])
new = self._saveVocab("new", [1, 6, 5, 3, 7])
mapping = checkpoint._get_vocabulary_mapping(old, new, "replace")
self.assertEqual(5 + 1, len(mapping)) # new + <unk>
self.assertAllEqual([0, -1, -1, 2, -1, 4], mapping)