# Hyper Parameters
embed_size = 128
hidden_size = 1024
num_layers = 1
num_epochs = 5
num_samples = 1000 # number of words to be sampled
batch_size = 20
seq_length = 30
learning_rate = 0.002
# Load Penn Treebank Dataset
train_path = './data/train.txt'
sample_path = './sample.txt'
corpus = Corpus()
ids = corpus.get_data(train_path, batch_size)
vocab_size = len(corpus.dictionary)
num_batches = ids.size(1) // seq_length
# RNN Based Language Model
class RNNLM(nn.Module):
def __init__(self, vocab_size, embed_size, hidden_size, num_layers):
super(RNNLM, self).__init__()
self.embed = nn.Embedding(vocab_size, embed_size)
self.lstm = nn.LSTM(embed_size,
hidden_size,
num_layers,
batch_first=True)
self.linear = nn.Linear(hidden_size, vocab_size)
self.init_weights()
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Hyper-parameters
embed_size = 128
hidden_size = 1024
num_layers = 1
num_epochs = 5
num_samples = 1000 # number of words to be sampled
batch_size = 20
seq_length = 30
learning_rate = 0.002
# Load "Penn Treebank" dataset
corpus = Corpus()
ids = corpus.get_data('data/train.txt', batch_size)
vocab_size = len(corpus.dictionary)
num_batches = ids.size(1) // seq_length
# RNN based language model
class RNNLM(nn.Module):
def __init__(self, vocab_size, embed_size, hidden_size, num_layers):
super(RNNLM, self).__init__()
self.embed = nn.Embedding(vocab_size, embed_size)
self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True)
self.linear = nn.Linear(hidden_size, vocab_size)
def forward(self, x, h):
# Embed word ids to vectors
x = self.embed(x)
# Hyper Parameters
embed_size = 128
hidden_size = 1024
num_layers = 1
num_epochs = 5
num_samples = 1000 # number of words to be sampled
batch_size = 20
seq_length = 30
learning_rate = 0.002
# Load Penn Treebank Dataset
train_path = './data/train.txt'
sample_path = './sample.txt'
corpus = Corpus()
ids = corpus.get_data(train_path, batch_size)
vocab_size = len(corpus.dictionary)
num_batches = ids.size(1) // seq_length
# RNN Based Language Model
class RNNLM(nn.Module):
def __init__(self, vocab_size, embed_size, hidden_size, num_layers):
super(RNNLM, self).__init__()
self.embed = nn.Embedding(vocab_size, embed_size)
self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True)
self.linear = nn.Linear(hidden_size, vocab_size)
self.init_weights()
def init_weights(self):
self.embed.weight.data.uniform_(-0.1, 0.1)
self.linear.bias.data.fill_(0)
embed_size = 128
hidden_size = 1024
num_layers = 1
num_epochs = 50
num_samples = 1000 # number of words to be sampled
batch_size = 20
seq_length = 30
learning_rate = 0.001
final_learning_rate = 0.0001
beta_1 = 0.9
beta_2 = 0.99
# Get dataset
corpus = Corpus()
ids = corpus.get_data('data/train.txt', batch_size)
vocab_size = len(corpus.dictionary)
num_batches = ids.size(1) // seq_length
def get_ckpt_name(model=model_choice, optimizer=optim_choice, lr=learning_rate, final_lr=final_learning_rate, momentum=momentum_choice,
beta1=beta_1, beta2=beta_2, gamma=gamma_choice):
name = {
'sgd': 'lr{}-momentum{}'.format(lr, momentum),
'adagrad': 'lr{}'.format(lr),
'adam': 'lr{}-betas{}-{}'.format(lr, beta1, beta2),
'amsgrad': 'lr{}-betas{}-{}'.format(lr, beta1, beta2),
'adabound': 'lr{}-betas{}-{}-final_lr{}-gamma{}'.format(lr, beta1, beta2, final_lr, gamma),
'amsbound': 'lr{}-betas{}-{}-final_lr{}-gamma{}'.format(lr, beta1, beta2, final_lr, gamma),
}[optimizer]
return '{}-{}-{}'.format(model, optimizer, name)
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Hyper-parameters
embed_size = 128
hidden_size = 1024
num_layers = 1
num_epochs = 5
num_samples = 10000 # number of words to be sampled
batch_size = 20
seq_length = 30
learning_rate = 0.002
# Load "Penn Treebank" dataset
corpus = Corpus()
ids = corpus.get_data('data/shakespeare.txt', batch_size)
vocab_size = len(corpus.dictionary)
num_batches = ids.size(1) // seq_length
infer_mode = False
model_path = "model.ckpt"
import sys
if len(sys.argv) > 1 and sys.argv[1] == 'infer':
infer_mode = True
if infer_mode:
print("Inference mode..")
else:
import torch
from torch import nn, optim
from torch.autograd import Variable
from data_utils import Corpus
seq_length = 30
train_file = "train.txt"
valid_file = "valid.txt"
test_file = "test.txt"
train_corpus = Corpus()
valid_corpus = Corpus()
test_corpus = Corpus()
train_id = train_corpus.get_data(train_file)
valid_id = valid_corpus.get_data(valid_file)
test_id = test_corpus.get_data(test_file)
vocab_size = len(train_corpus.dic)
num_batches = train_id.size(1) // seq_length
class languagemodel(nn.Module):
def __init__(self, vocab_size, embed_dim, hidden_size, num_layers):
super(languagemodel, self).__init__()
self.embed = nn.Embedding(vocab_size, embed_dim)
self.lstm = nn.LSTM(embed_dim,
hidden_size,
num_layers,
batch_first=True)
# Hyper-parameters
embed_size = 220
hidden_size = 220
num_layers = 2
num_epochs = 10
num_samples = 30 # number of words to be sampled
batch_size = 20
seq_length = 30
dropout = 0.3
learning_rate = 0.005
seed = 621
date = datetime.datetime.now().strftime("%d-%m-%y %H-%M-%S")
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
corpus = Corpus()
ids = corpus.get_data('data/train.txt', batch_size) # divide to batch size
valid_d = corpus.get_data('data/valid.txt', batch_size)
test_d = corpus.get_data('data/test.txt', batch_size)
vocab_size = len(corpus.dictionary)
num_batches = ids.size(1) // seq_length
model = HW3.RNNLM(vocab_size, embed_size, hidden_size, num_layers, dropout)
model.load_state_dict(torch.load(model_name,map_location=device))
model.eval()
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
model.cuda()
else:
model.cpu()
from data_utils import Dictionary, Corpus
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
embed_size = 128
hidden_size = 1024
num_layers = 1
num_epochs = 0
num_samples = 1000
batch_size = 20
seq_length = 30
learning_rate = 0.002
corpus = Corpus()
ids = corpus.get_data('data/train.txt',
batch_size) # (batch_size, word_ids from many sentence)
vocab_size = len(corpus.dictionary)
num_batches = ids.size(1) // seq_length
class RNNLM(nn.Module):
def __init__(self, vocab_size, embed_size, hidden_size, num_layers):
super(RNNLM, self).__init__()
self.embed = nn.Embedding(vocab_size, embed_size)
self.lstm = nn.LSTM(embed_size,
hidden_size,
num_layers,
batch_first=True)
self.linear = nn.Linear(hidden_size, vocab_size)
def forward(self, x, h):
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Hyper-parameters
embed_size = 128
hidden_size = 1024
num_layers = 1
num_epochs = 5
num_samples = 1000 # number of words to be sampled
batch_size = 10
seq_length = 30
learning_rate = 0.002
# Load "Penn Treebank" dataset
corpus = Corpus()
ids = corpus.get_data('data/short.txt',
batch_size) # label each word with word ids
print('\n\n\n\n')
vocab_size = len(corpus.dictionary)
num_batches = ids.size(1) // seq_length
# RNN based language model
class RNNLM(nn.Module):
def __init__(self, vocab_size, embed_size, hidden_size, num_layers):
super(RNNLM, self).__init__()
self.embed = nn.Embedding(vocab_size, embed_size)
self.lstm = nn.LSTM(embed_size,
hidden_size,
num_layers,
batch_first=True)
self.linear = nn.Linear(hidden_size, vocab_size)