def __init__(self, model: SiameseModel, batch_size: int, num_context_turns: int = 1, ranking: bool = True, attention: bool = False, responses: SimpleVocabulary = None, preproc_func: Callable = None, interact_pred_num: int = 3, *args, **kwargs) -> None: super().__init__() self.batch_size = batch_size self.num_context_turns = num_context_turns self.ranking = ranking self.attention = attention self.preproc_responses = [] self.response_embeddings = None self.preproc_func = preproc_func self.interact_pred_num = interact_pred_num self.model = model if self.ranking: self.responses = {el[1]: el[0] for el in responses.items()} self._build_preproc_responses() if not self.attention: self._build_response_embeddings()
def __init__(self, model: SiameseModel, batch_size: int, num_context_turns: int = 1, ranking: bool = True, attention: bool = False, responses: SimpleVocabulary = None, preproc_func: Callable = None, interact_pred_num: int = 3, *args, **kwargs) -> None: super().__init__() self.batch_size = batch_size self.num_context_turns = num_context_turns self.ranking = ranking self.attention = attention self.preproc_responses = [] self.response_embeddings = None self.preproc_func = preproc_func self.interact_pred_num = interact_pred_num self.model = model if self.ranking: self.responses = {el[1]: el[0] for el in responses.items()} self._build_preproc_responses() if not self.attention: self._build_response_embeddings()
tokenizer(['Kaggle is the best place to study machine learning.']) train_x_lower_tokenized = str_lower(tokenizer(train_iterator.get_instances(data_type='train')[0])) """##Vocabulary""" # initialize simple vocabulary to collect all appeared in the dataset classes classes_vocab = SimpleVocabulary( save_path='./tmp/classes.dict', load_path='./tmp/classes.dict') classes_vocab.fit((train_iterator.get_instances(data_type='train')[1])) classes_vocab.save() # show classes list(classes_vocab.items()) # also one can collect vocabulary of textual tokens appeared 2 and more times in the dataset token_vocab = SimpleVocabulary( save_path='./tmp/tokens.dict', load_path='./tmp/tokens.dict', min_freq=2, special_tokens=('<PAD>', '<UNK>',), unk_token='<UNK>') token_vocab.fit(train_x_lower_tokenized) token_vocab.save() # number of tokens in dictionary len(token_vocab)
for x, y in list(zip(x_train, y_train))[:3]: print('x:', x) print('y:', y) print('=================') # tokenize all input data tokenizer = NLTKMosesTokenizer() train_x_lower_tokenized = str_lower( tokenizer(train_iterator.get_instances(data_type='train')[0])) # get the intent categories classes_vocab = SimpleVocabulary(save_path='./tmp/classes.dict', load_path='./tmp/classes.dict') classes_vocab.fit(train_iterator.get_instances(data_type='train')[1]) classes_vocab.save() print(list(classes_vocab.items())) # display classes # get all token vocab token_vocab = SimpleVocabulary(save_path='./tmp/tokens.dict', load_path='./tmp/tokens.dict') token_vocab.fit(train_x_lower_tokenized) token_vocab.save() # we will use GLOVE embedding if not os.path.isfile("./glove.6B.100d.txt"): simple_download( url="http://files.deeppavlov.ai/embeddings/glove.6B.100d.txt", destination="./glove.6B.100d.txt") embedder = GloVeEmbedder(load_path='./glove.6B.100d.txt', dim=100, pad_zero=True)