def test_uppercase_tokens(self): """ Test that the text is not lower-cased when lowercase_tokens is False. """ custom_dh = CompositionalDataHandler.from_config( CompositionalDataHandler.Config(), FeatureConfig(word_feat=WordFeatConfig(vocab_from_all_data=True, min_freq=1)), featurizer=SimpleFeaturizer.from_config( SimpleFeaturizer.Config(lowercase_tokens=False), FeatureConfig()), ) custom_dh.init_metadata_from_raw_data(self.train_data, self.eval_data, self.test_data) self.assertSetEqual( set(custom_dh.features["word_feat"].vocab.stoi), { "<unk>", "What", "EVENTS", "can", "I", "go", "today", "Are", "there", "any", "adult", "events", "this", "weekend", }, )
def test_min_freq(self): """ Test that UNKification is triggered when min_freq is 2. """ custom_dh = CompositionalDataHandler.from_config( CompositionalDataHandler.Config(), FeatureConfig(word_feat=WordFeatConfig(vocab_from_all_data=True, min_freq=2)), featurizer=SimpleFeaturizer.from_config( SimpleFeaturizer.Config(lowercase_tokens=True), FeatureConfig()), ) custom_dh.init_metadata_from_raw_data(self.train_data, self.eval_data, self.test_data) # <unk>-NUM = <unk> for numeric tokens self.assertSetEqual( set(custom_dh.features["word_feat"].vocab.stoi), {"<unk>", "<unk>-NUM", "<unk>", "<unk>", "events"}, )
def setUp(self): self.train_data = [{ DFColumn.DOC_LABEL: "IN:GET_EVENT", DFColumn.WORD_LABEL: [{ "id": "SL:DATE_TIME", "span": { "start": 21, "end": 26 }, "text": "today", }], DFColumn.UTTERANCE: "What EVENTS can I go today", DFColumn.DICT_FEAT: "", DFColumn.SEQLOGICAL: "[IN:GET_EVENT What EVENTS can I go [SL:DATE_TIME today ] ]", }] self.eval_data = [{ DFColumn.DOC_LABEL: "IN:GET_EVENT", DFColumn.WORD_LABEL: [ { "id": "SL:ATTRIBUTE_EVENT", "span": { "start": 14, "end": 19 }, "text": "adult", }, { "id": "SL:DATE_TIME", "span": { "start": 27, "end": 39 }, "text": "this weekend", }, ], DFColumn.UTTERANCE: "Are there any adult events this weekend", DFColumn.DICT_FEAT: "", DFColumn.SEQLOGICAL: "[IN:GET_EVENT Are there any [SL:ATTRIBUTE_EVENT adult ] events [SL:DATE_TIME this weekend ] ]", }] self.test_data = [{ DFColumn.DOC_LABEL: "IN:GET_INFO_ROAD_CONDITION", DFColumn.WORD_LABEL: [ { "id": "SL:ROAD_CONDITION", "span": { "start": 9, "end": 21 }, "text": "any flooding", }, { "id": "SL:DESTINATION", "span": { "start": 36, "end": 41 }, "text": "Karen", "subframe": { "utterance": "Karen", "domain": "", "intent": "IN:GET_LOCATION_HOME", "slots": [{ "id": "SL:CONTACT", "span": { "start": 0, "end": 5 }, "text": "Karen", }], "span": { "start": 0, "end": 5 }, }, }, ], DFColumn.UTTERANCE: "Is there any flooding on the way to Karen's?", DFColumn.DICT_FEAT: "", DFColumn.SEQLOGICAL: "[IN:GET_INFO_ROAD_CONDITION Is there [SL:ROAD_CONDITION any flooding ] on the way to [SL:DESTINATION [IN:GET_LOCATION_HOME [SL:CONTACT Karen 's ? ] ] ] ]", }] self.dh = CompositionalDataHandler.from_config( CompositionalDataHandler.Config(), FeatureConfig(word_feat=WordFeatConfig(vocab_from_all_data=True, min_freq=1)), featurizer=SimpleFeaturizer.from_config( SimpleFeaturizer.Config(lowercase_tokens=True), FeatureConfig()), )