def __init__(self, model: SiameseModel, batch_size: int, num_context_turns: int = 1, ranking: bool = True, attention: bool = False, responses: SimpleVocabulary = None, preproc_func: Callable = None, interact_pred_num: int = 3, *args, **kwargs) -> None: super().__init__() self.batch_size = batch_size self.num_context_turns = num_context_turns self.ranking = ranking self.attention = attention self.preproc_responses = [] self.response_embeddings = None self.preproc_func = preproc_func self.interact_pred_num = interact_pred_num self.model = model if self.ranking: self.responses = {el[1]: el[0] for el in responses.items()} self._build_preproc_responses() if not self.attention: self._build_response_embeddings()
def __init__(self, model: SiameseModel, batch_size: int, num_context_turns: int = 1, ranking: bool = True, attention: bool = False, responses: SimpleVocabulary = None, preproc_func: Callable = None, interact_pred_num: int = 3, *args, **kwargs) -> None: super().__init__() self.batch_size = batch_size self.num_context_turns = num_context_turns self.ranking = ranking self.attention = attention self.preproc_responses = [] self.response_embeddings = None self.preproc_func = preproc_func self.interact_pred_num = interact_pred_num self.model = model if self.ranking: self.responses = {el[1]: el[0] for el in responses.items()} self._build_preproc_responses() if not self.attention: self._build_response_embeddings()
def __init__(self, data: Dict[str, List[Union[str, Path]]], load_path: Union[str, Path], seed: Optional[int] = None, shuffle: bool = True, unroll_steps: Optional[int] = None, n_gpus: Optional[int] = None, max_word_length: Optional[int] = None, bos: str = "<S>", eos: str = "</S>", *args, **kwargs) -> None: self.unroll_steps = unroll_steps self.n_gpus = n_gpus self.bos = bos self.eos = eos self.str_utf8_encoder = StrUTF8Encoder( max_word_length=max_word_length, pad_special_char_use=True, word_boundary_special_char_use=True, sentence_boundary_special_char_use=False, reversed_sentense_tokens=False, bos=self.bos, eos=self.eos, save_path=load_path, load_path=load_path, ) self.simple_vocab = SimpleVocabulary( min_freq=2, special_tokens=[self.eos, self.bos, "<UNK>"], unk_token="<UNK>", freq_drop_load=True, save_path=load_path, load_path=load_path, ) super().__init__(data, seed, shuffle, *args, **kwargs)
import numpy as np import torch import torch.cuda as cuda from torch.nn.functional import softmax from torch.utils.data import DataLoader from deeppavlov.core.data.simple_vocab import SimpleVocabulary from Task_2_work_ver.Task_1_character_lm.plot_loss import plot_loss from Task_2_work_ver.Task_1_character_lm.get_func import read_infile, Dataset, Padder, Config base_path = r'C:\Users\Andrey' experiments_path = r'C:\Users\Andrey\Google Диск\courses\DeepPavlov\Task-2-preduct0r\data\Task_1' train_words = read_infile(os.path.join(base_path, "russian-train-high")) test_words = read_infile(os.path.join(base_path, "russian-test")) vocab = SimpleVocabulary(special_tokens=('PAD', 'UNK', 'BEGIN', 'END'), unk_token='UNK', save_path=experiments_path) vocab.fit([list(x) for x in train_words]) config = Config(lr=0.0001, batch_size=512, num_epochs=1000) net = torch.load(os.path.join(experiments_path, "net.pb")) if cuda.is_available(): device = 'cuda' else: device = 'cpu' # ============================================================================ # Write a function predict_on_batch that outputs letter probabilities of all words in the batch.
# check str_lower(['Kaggle is the best place to study machine learning.']) """##Tokenizer""" tokenizer = NLTKMosesTokenizer() # check tokenizer(['Kaggle is the best place to study machine learning.']) train_x_lower_tokenized = str_lower(tokenizer(train_iterator.get_instances(data_type='train')[0])) """##Vocabulary""" # initialize simple vocabulary to collect all appeared in the dataset classes classes_vocab = SimpleVocabulary( save_path='./tmp/classes.dict', load_path='./tmp/classes.dict') classes_vocab.fit((train_iterator.get_instances(data_type='train')[1])) classes_vocab.save() # show classes list(classes_vocab.items()) # also one can collect vocabulary of textual tokens appeared 2 and more times in the dataset token_vocab = SimpleVocabulary( save_path='./tmp/tokens.dict', load_path='./tmp/tokens.dict', min_freq=2, special_tokens=('<PAD>', '<UNK>',), unk_token='<UNK>')
import warnings warnings.filterwarnings("ignore") import sys sys.path.append('/rapids/notebooks/my_data/BMSTU_hack/') import torch import biGRU_model from deeppavlov.core.data.simple_vocab import SimpleVocabulary import numpy as np gru = torch.load('/rapids/notebooks/my_data/BMSTU_hack/models/biGRU') device = torch.device('cpu') vocab = SimpleVocabulary( save_path="/rapids/notebooks/my_data/BMSTU_hack/models/vocab.dict") gru = biGRU_model.BiGRU(vocab.count, embedding_dim=10, hidden_size=50, device='cpu') gru.load_state_dict( torch.load('/rapids/notebooks/my_data/BMSTU_hack/models/biGRU', map_location=device)) from tbot import config import telebot bot = telebot.TeleBot(config.token) @bot.message_handler(content_types=["text"]) def repeat_all_messages(message):
from deeppavlov.models.bert.bert_classifier import BertClassifierModel from deeppavlov.metrics.accuracy import sets_accuracy reader = BasicClassificationDatasetReader() data = reader.read(data_path="./stanfordSentimentTreebank", train="/content/train.csv", valid="/content/valid.csv", test="/content/test1.csv", x="original", y="meanGrade") iterator = BasicClassificationDatasetIterator(data, seed=42, shuffle=True) bert_preprocessor = BertPreprocessor( vocab_file= "~/.deeppavlov/downloads/bert_models/cased_L-12_H-768_A-12/vocab.txt", do_lower_case=False, max_seq_length=64) vocab = SimpleVocabulary(save_path="./binary_classes.dict") iterator.get_instances(data_type="train") vocab.fit(iterator.get_instances(data_type="train")[1]) one_hotter = OneHotter(depth=vocab.len, single_vector=True) prob2labels = Proba2Labels(max_proba=True) bert_classifier = BertClassifierModel( n_classes=vocab.len, return_probas=True, one_hot_labels=True, bert_config_file= "~/.deeppavlov/downloads/bert_models/cased_L-12_H-768_A-12/bert_config.json", pretrained_bert= "~/.deeppavlov/downloads/bert_models/cased_L-12_H-768_A-12/bert_model.ckpt", save_path="sst_bert_model/model", load_path="sst_bert_model/model", keep_prob=0.5,
dr = BasicClassificationDatasetReader().read(data_path='./', train='train.csv', valid='valid.csv', test='test.csv', x='original', y='meanGrade') train_iterator = BasicClassificationDatasetIterator(data=dr, seed=42) x_train, y_train = train_iterator.get_instances(data_type='train') for x, y in list(zip(x_train, y_train))[:5]: print('x:', x) print('y:', y) print('=================') tokenizer = NLTKMosesTokenizer() train_x_lower_tokenized = str_lower( tokenizer(train_iterator.get_instances(data_type='train')[0])) classes_vocab = SimpleVocabulary(save_path='./snips/classes.dict', load_path='./snips/classes.dict') vocab = SimpleVocabulary(save_path="./binary_classes.dict") classes_vocab.fit((train_iterator.get_instances(data_type='train')[1])) classes_vocab.save() token_vocab = SimpleVocabulary(save_path='./snips/tokens.dict', load_path='./snips/tokens.dict', min_freq=2, special_tokens=( '<PAD>', '<UNK>', ), unk_token='<UNK>') token_vocab.fit(train_x_lower_tokenized) token_vocab.save() token_vocab.freqs.most_common()[:10] tfidf = SklearnComponent(
#print([(k, len(dr[k])) for k in dr.keys()]) # print a few x, y pairs x_train, y_train = train_iterator.get_instances(data_type='train') for x, y in list(zip(x_train, y_train))[:3]: print('x:', x) print('y:', y) print('=================') # tokenize all input data tokenizer = NLTKMosesTokenizer() train_x_lower_tokenized = str_lower( tokenizer(train_iterator.get_instances(data_type='train')[0])) # get the intent categories classes_vocab = SimpleVocabulary(save_path='./tmp/classes.dict', load_path='./tmp/classes.dict') classes_vocab.fit(train_iterator.get_instances(data_type='train')[1]) classes_vocab.save() print(list(classes_vocab.items())) # display classes # get all token vocab token_vocab = SimpleVocabulary(save_path='./tmp/tokens.dict', load_path='./tmp/tokens.dict') token_vocab.fit(train_x_lower_tokenized) token_vocab.save() # we will use GLOVE embedding if not os.path.isfile("./glove.6B.100d.txt"): simple_download( url="http://files.deeppavlov.ai/embeddings/glove.6B.100d.txt", destination="./glove.6B.100d.txt")
words.append(temp[1]) return words #================================================== train_words = read_infile(os.path.join(base_path, "russian-train-high")) dev_words = read_infile(os.path.join(base_path, "russian-dev")) test_words = read_infile(os.path.join(base_path, "russian-test")) print(len(train_words), len(dev_words), len(test_words)) print(*train_words[:10]) #================================================== vocab = SimpleVocabulary( special_tokens=('PAD', 'UNK', 'BEGIN', 'END'), unk_token='UNK', save_path=r'C:\Users\Andrey\Google Диск\courses\DeepPavlov\Task-2-preduct0r' ) vocab.fit([list(x) for x in train_words]) #================================================== train_dataset = Dataset(train_words, vocab) dev_dataset = Dataset(dev_words, vocab) test_dataset = Dataset(test_words, vocab) #================================================== train_batcher = DataLoader(train_dataset, batch_size=1) dev_batcher = DataLoader(dev_dataset, batch_size=1) test_batcher = DataLoader(test_dataset, batch_size=1) # for i, (items, classes) in enumerate(train_batcher):