# Hyper Parameters embed_size = 128 hidden_size = 1024 num_layers = 1 num_epochs = 5 num_samples = 1000 # number of words to be sampled batch_size = 20 seq_length = 30 learning_rate = 0.002 # Load Penn Treebank Dataset train_path = './data/train.txt' sample_path = './sample.txt' corpus = Corpus() ids = corpus.get_data(train_path, batch_size) vocab_size = len(corpus.dictionary) num_batches = ids.size(1) // seq_length # RNN Based Language Model class RNNLM(nn.Module): def __init__(self, vocab_size, embed_size, hidden_size, num_layers): super(RNNLM, self).__init__() self.embed = nn.Embedding(vocab_size, embed_size) self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True) self.linear = nn.Linear(hidden_size, vocab_size) self.init_weights()
# Device configuration device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Hyper-parameters embed_size = 128 hidden_size = 1024 num_layers = 1 num_epochs = 5 num_samples = 1000 # number of words to be sampled batch_size = 20 seq_length = 30 learning_rate = 0.002 # Load "Penn Treebank" dataset corpus = Corpus() ids = corpus.get_data('data/train.txt', batch_size) vocab_size = len(corpus.dictionary) num_batches = ids.size(1) // seq_length # RNN based language model class RNNLM(nn.Module): def __init__(self, vocab_size, embed_size, hidden_size, num_layers): super(RNNLM, self).__init__() self.embed = nn.Embedding(vocab_size, embed_size) self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True) self.linear = nn.Linear(hidden_size, vocab_size) def forward(self, x, h): # Embed word ids to vectors x = self.embed(x)
# Hyper Parameters embed_size = 128 hidden_size = 1024 num_layers = 1 num_epochs = 5 num_samples = 1000 # number of words to be sampled batch_size = 20 seq_length = 30 learning_rate = 0.002 # Load Penn Treebank Dataset train_path = './data/train.txt' sample_path = './sample.txt' corpus = Corpus() ids = corpus.get_data(train_path, batch_size) vocab_size = len(corpus.dictionary) num_batches = ids.size(1) // seq_length # RNN Based Language Model class RNNLM(nn.Module): def __init__(self, vocab_size, embed_size, hidden_size, num_layers): super(RNNLM, self).__init__() self.embed = nn.Embedding(vocab_size, embed_size) self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True) self.linear = nn.Linear(hidden_size, vocab_size) self.init_weights() def init_weights(self): self.embed.weight.data.uniform_(-0.1, 0.1) self.linear.bias.data.fill_(0)
embed_size = 128 hidden_size = 1024 num_layers = 1 num_epochs = 50 num_samples = 1000 # number of words to be sampled batch_size = 20 seq_length = 30 learning_rate = 0.001 final_learning_rate = 0.0001 beta_1 = 0.9 beta_2 = 0.99 # Get dataset corpus = Corpus() ids = corpus.get_data('data/train.txt', batch_size) vocab_size = len(corpus.dictionary) num_batches = ids.size(1) // seq_length def get_ckpt_name(model=model_choice, optimizer=optim_choice, lr=learning_rate, final_lr=final_learning_rate, momentum=momentum_choice, beta1=beta_1, beta2=beta_2, gamma=gamma_choice): name = { 'sgd': 'lr{}-momentum{}'.format(lr, momentum), 'adagrad': 'lr{}'.format(lr), 'adam': 'lr{}-betas{}-{}'.format(lr, beta1, beta2), 'amsgrad': 'lr{}-betas{}-{}'.format(lr, beta1, beta2), 'adabound': 'lr{}-betas{}-{}-final_lr{}-gamma{}'.format(lr, beta1, beta2, final_lr, gamma), 'amsbound': 'lr{}-betas{}-{}-final_lr{}-gamma{}'.format(lr, beta1, beta2, final_lr, gamma), }[optimizer] return '{}-{}-{}'.format(model, optimizer, name)
# Device configuration device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Hyper-parameters embed_size = 128 hidden_size = 1024 num_layers = 1 num_epochs = 5 num_samples = 10000 # number of words to be sampled batch_size = 20 seq_length = 30 learning_rate = 0.002 # Load "Penn Treebank" dataset corpus = Corpus() ids = corpus.get_data('data/shakespeare.txt', batch_size) vocab_size = len(corpus.dictionary) num_batches = ids.size(1) // seq_length infer_mode = False model_path = "model.ckpt" import sys if len(sys.argv) > 1 and sys.argv[1] == 'infer': infer_mode = True if infer_mode: print("Inference mode..") else:
import torch from torch import nn, optim from torch.autograd import Variable from data_utils import Corpus seq_length = 30 train_file = "train.txt" valid_file = "valid.txt" test_file = "test.txt" train_corpus = Corpus() valid_corpus = Corpus() test_corpus = Corpus() train_id = train_corpus.get_data(train_file) valid_id = valid_corpus.get_data(valid_file) test_id = test_corpus.get_data(test_file) vocab_size = len(train_corpus.dic) num_batches = train_id.size(1) // seq_length class languagemodel(nn.Module): def __init__(self, vocab_size, embed_dim, hidden_size, num_layers): super(languagemodel, self).__init__() self.embed = nn.Embedding(vocab_size, embed_dim) self.lstm = nn.LSTM(embed_dim, hidden_size, num_layers, batch_first=True)
# Hyper-parameters embed_size = 220 hidden_size = 220 num_layers = 2 num_epochs = 10 num_samples = 30 # number of words to be sampled batch_size = 20 seq_length = 30 dropout = 0.3 learning_rate = 0.005 seed = 621 date = datetime.datetime.now().strftime("%d-%m-%y %H-%M-%S") device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') corpus = Corpus() ids = corpus.get_data('data/train.txt', batch_size) # divide to batch size valid_d = corpus.get_data('data/valid.txt', batch_size) test_d = corpus.get_data('data/test.txt', batch_size) vocab_size = len(corpus.dictionary) num_batches = ids.size(1) // seq_length model = HW3.RNNLM(vocab_size, embed_size, hidden_size, num_layers, dropout) model.load_state_dict(torch.load(model_name,map_location=device)) model.eval() torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed(seed) model.cuda() else: model.cpu()
from data_utils import Dictionary, Corpus # Device configuration device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') embed_size = 128 hidden_size = 1024 num_layers = 1 num_epochs = 0 num_samples = 1000 batch_size = 20 seq_length = 30 learning_rate = 0.002 corpus = Corpus() ids = corpus.get_data('data/train.txt', batch_size) # (batch_size, word_ids from many sentence) vocab_size = len(corpus.dictionary) num_batches = ids.size(1) // seq_length class RNNLM(nn.Module): def __init__(self, vocab_size, embed_size, hidden_size, num_layers): super(RNNLM, self).__init__() self.embed = nn.Embedding(vocab_size, embed_size) self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True) self.linear = nn.Linear(hidden_size, vocab_size) def forward(self, x, h):
# Device configuration device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Hyper-parameters embed_size = 128 hidden_size = 1024 num_layers = 1 num_epochs = 5 num_samples = 1000 # number of words to be sampled batch_size = 10 seq_length = 30 learning_rate = 0.002 # Load "Penn Treebank" dataset corpus = Corpus() ids = corpus.get_data('data/short.txt', batch_size) # label each word with word ids print('\n\n\n\n') vocab_size = len(corpus.dictionary) num_batches = ids.size(1) // seq_length # RNN based language model class RNNLM(nn.Module): def __init__(self, vocab_size, embed_size, hidden_size, num_layers): super(RNNLM, self).__init__() self.embed = nn.Embedding(vocab_size, embed_size) self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True) self.linear = nn.Linear(hidden_size, vocab_size)