def sample(model: CharRNN, vocab: Vocab) -> str:
model.eval()
with torch.no_grad():
hidden = None
input = torch.from_numpy(
np.array([
convert_idx_to_one_hot(vocab, vocab.char_to_idx(SOS_TOKEN))
])).unsqueeze(0)
input = input.float().to(config.device)
pred = ''
for _ in range(8):
out, hidden = model(input, hidden)
if out.view(-1).div(0.8).exp().sum() == 0:
continue
topi = torch.multinomial(out.view(-1).div(0.8).exp(), 1)[0]
if topi.item() == vocab.char_to_idx(EOS_TOKEN):
break
pred += vocab.idx_to_char(topi.item())
input = torch.from_numpy(
np.array([convert_idx_to_one_hot(vocab,
topi.item())])).unsqueeze(0)
input = input.float().to(config.device)
return pred
import numpy as np
import torch
from flask import Flask, request, jsonify, render_template
from torch.nn import functional as F
from model import CharRNN
from settings import *
from utils import load_dict, create_tune_header
app = Flask(__name__)
print("Environment:", app.config["ENV"])
# Create and load model
model = CharRNN(n_char)
model.load_state_dict(torch.load(default_model_path, map_location='cpu'))
model.eval()
# Load necessary dictionaries
int2char = load_dict(int2char_path)
char2int = load_dict(char2int_path)
counter = Value("i", 0)
error_message = "We created some tunes, but it seems like we can't create music from these melodies.Please try again!"
print("Ready!")
@app.route("/")
def generate_song():
return render_template("index.html")
class Trainer(object):
def __init__(self, args):
self.args = args
self.device = torch.device('cuda' if self.args.cuda else 'cpu')
self.convert = None
self.model = None
self.optimizer = None
self.criterion = self.get_loss
self.meter = AverageValueMeter()
self.train_loader = None
self.get_data()
self.get_model()
self.get_optimizer()
def get_data(self):
self.convert = TextConverter(self.args.txt,
max_vocab=self.args.max_vocab)
dataset = TextDataset(self.args.txt, self.args.len,
self.convert.text_to_arr)
self.train_loader = DataLoader(dataset,
self.args.batch_size,
shuffle=True,
num_workers=self.args.num_workers)
def get_model(self):
self.model = CharRNN(self.convert.vocab_size, self.args.embed_dim,
self.args.hidden_size, self.args.num_layers,
self.args.dropout, self.args.cuda).to(self.device)
if self.args.cuda:
cudnn.benchmark = True
def get_optimizer(self):
optimizer = torch.optim.Adam(self.model.parameters(), lr=self.args.lr)
self.optimizer = ScheduledOptim(optimizer)
@staticmethod
def get_loss(score, label):
return nn.CrossEntropyLoss()(score, label.view(-1))
def save_checkpoint(self, epoch):
if (epoch + 1) % self.args.save_interval == 0:
model_out_path = self.args.save_file + "epoch_{}_model.pth".format(
epoch + 1)
torch.save(self.model, model_out_path)
print("Checkpoint saved to {}".format(model_out_path))
def save(self):
model_out_path = self.args.save_file + "final_model.pth"
torch.save(self.model, model_out_path)
print("Final model saved to {}".format(model_out_path))
@staticmethod
def pick_top_n(predictions, top_n=5):
top_predict_prob, top_predict_label = torch.topk(predictions, top_n, 1)
top_predict_prob /= torch.sum(top_predict_prob)
top_predict_prob = top_predict_prob.squeeze(0).cpu().numpy()
top_predict_label = top_predict_label.squeeze(0).cpu().numpy()
c = np.random.choice(top_predict_label, size=1, p=top_predict_prob)
return c
def train(self):
self.meter.reset()
self.model.train()
for x, y in tqdm(self.train_loader):
y = y.long()
x, y = x.to(self.device), y.to(self.device)
# Forward.
score, _ = self.model(x)
loss = self.criterion(score, y)
# Backward.
self.optimizer.zero_grad()
loss.backward()
# Clip gradient.
nn.utils.clip_grad_norm_(self.model.parameters(), 5)
self.optimizer.step()
self.meter.add(loss.item())
print('perplexity: {}'.format(np.exp(self.meter.value()[0])))
def test(self):
self.model.eval()
begin = np.array([i for i in self.args.begin])
begin = np.random.choice(begin, size=1)
text_len = self.args.predict_len
samples = [self.convert.word_to_int(c) for c in begin]
input_txt = torch.LongTensor(samples)[None]
input_txt = input_txt.to(self.device)
_, init_state = self.model(input_txt)
result = samples
model_input = input_txt[:, -1][:, None]
with torch.no_grad():
for i in range(text_len):
out, init_state = self.model(model_input, init_state)
prediction = self.pick_top_n(out.data)
model_input = torch.LongTensor(prediction)[None].to(
self.device)
result.append(prediction[0])
print(self.convert.arr_to_text(result))
def predict(self):
self.model.eval()
samples = [self.convert.word_to_int(c) for c in self.args.begin]
input_txt = torch.LongTensor(samples)[None].to(self.device)
_, init_state = self.model(input_txt)
result = samples
model_input = input_txt[:, -1][:, None]
with torch.no_grad():
for i in range(self.args.predict_len):
out, init_state = self.model(model_input, init_state)
prediction = self.pick_top_n(out.data)
model_input = torch.LongTensor(prediction)[None].to(
self.device)
result.append(prediction[0])
print(self.convert.arr_to_text(result))
def run(self):
for e in range(self.args.max_epoch):
print('===> EPOCH: {}/{}'.format(e + 1, self.args.max_epoch))
self.train()
self.test()
self.save_checkpoint(e)
self.save()
def train(opt, x_train, x_val, dictionary_len):
''' Training a network
Arguments
---------
net: CharRNN network
data: training data to train the network (text)
epochs: Number of epochs to train
batch_size: Number of mini-sequences per mini-batch, aka batch size
seq_length: Number of character steps per mini-batch
lr: learning rate
clip: gradient clipping
val_frac: Fraction of data to hold out for validation
print_every: Number of steps for printing training and validation loss
'''
torch.manual_seed(0)
np.random.seed(0)
random.seed(0)
# Declaring the hyperparameters
batch_size = opt.batch_size
seq_length = int(opt.seq_length)
epochs = 50
if torch.cuda.is_available():
device = "cuda"
torch.cuda.manual_seed_all(0)
else:
device = "cpu"
print(device)
date = datetime.now().strftime('%y%m%d%H%M%S')
if opt.nologs:
writer = SummaryWriter(log_dir=f'logs/nologs/')
else:
writer = SummaryWriter(log_dir=f'logs/logs_{date}/')
y_train = get_labels_text_prediction(x_train)
train_dataset = TextDataset(x_train, y_train, max_len=seq_length)
if not opt.onlytrain:
y_val = get_labels_text_prediction(x_val)
val_dataset = TextDataset(x_val, y_val, max_len=seq_length)
val_loader = DataLoader(dataset=val_dataset,
pin_memory=device == 'cuda',
batch_size=batch_size,
shuffle=False)
train_loader = DataLoader(dataset=train_dataset,
pin_memory=device == 'cuda',
batch_size=batch_size,
shuffle=True)
model_params = {
'dictionary_len': dictionary_len,
'dropout': opt.dropout,
'hidden_size': opt.hidden_size,
'layers': opt.layers,
'embedding_len': 32,
'device': device,
'lr': opt.lr
}
model = CharRNN(**model_params).to(device)
print(model)
# embed()
# summary(model, input_size=(channels, H, W))
# summary(model, input_size=(dictionary_len, 28, 28))
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.CrossEntropyLoss()
if opt.scheduler:
scheduler = ReduceLROnPlateau(optimizer,
'min',
cooldown=3,
factor=0.5,
patience=10)
global_step = 0
for j in trange(epochs, desc='T raining LSTM...'):
for i, (x, y) in enumerate(train_loader):
if i == len(train_loader) - 1:
print("FER PADDING - DE MOMENT NO VA")
continue
model.train()
x = x.to(device)
y = y.to(device)
# state_h, state_c = model.zero_state(opt.batch_size)
# # Transfer data to GPU
# state_h = state_h.to(device)
# state_c = state_c.to(device)
# DELETE PAST GRADIENTS
optimizer.zero_grad()
# FORWARD PASS --> ultim state , (tots) [ state_h[-1] == pred ]
pred, (state_h, state_c) = model(x)
# pred, (state_h, state_c) = model(x, (state_h, state_c))
# CALCULATE LOSS
# pred = pred.transpose(1, 2)
pred2 = pred.view(-1, dictionary_len)
y2 = y.view(-1)
loss = criterion(pred2, y2)
loss_value = loss.item()
# BACKWARD PASS
loss.backward()
# MINIMIZE LOSS
optimizer.step()
global_step += 1
if i % 100 == 0:
writer.add_scalar('train/loss', loss_value, global_step)
print('[Training epoch {}: {}/{}] Loss: {}'.format(
j, i, len(train_loader), loss_value))
if not opt.onlytrain:
val_loss = []
for i, (x, y) in enumerate(val_loader):
if i == len(val_loader) - 1:
# print("FER PADDING - DE MOMENT NO VA")
continue
model.eval()
x = x.to(device)
y = y.to(device)
# state_h, state_c = model.zero_state(opt.batch_size)
# state_h = state_h.to(device)
# state_c = state_c.to(device)
# NO BACKPROPAGATION
# FORWARD PASS
# pred, (state_h, state_c) = model(x, (state_h, state_c))
pred, (state_h, state_c) = model(x)
# CALCULATE LOSS
# pred = pred.transpose(1, 2)
# pred = [batch x 40 x diccionary_len]
# y = [batch x 40]
pred2 = pred.view(-1, dictionary_len)
y2 = y.view(-1)
loss = criterion(pred2, y2)
# loss = criterion(pred, y)
val_loss.append(loss.item())
if i % 50 == 0:
print('[Validation epoch {}: {}/{}] Loss: {}'.format(
j, i, len(val_loader), loss.item()))
writer.add_scalar('val/loss', np.mean(val_loss), j)
if opt.scheduler:
scheduler.step(np.mean(val_loss))
writer.add_scalar("lr", optimizer.param_groups[0]["lr"], j)
predicted_words = inference_prediction(model, device, 500)
# output = pred[0].unsqueeze(0) # [1,diccionary_len, 40]
# predicted_words = do_inference_test(output, model, device)
print(predicted_words)
writer.add_text('val/Generated_Samples', predicted_words, j)
checkpoint = {
"state_dict": model.state_dict(),
"optimizer": optimizer.state_dict(),
}
# if j % 5 == 0:
os.makedirs("weights/{}".format(date), exist_ok=True)
torch.save(checkpoint, "weights/{}/checkpoint_{}.pt".format(date, j))
