def sample(model: CharRNN, vocab: Vocab) -> str: model.eval() with torch.no_grad(): hidden = None input = torch.from_numpy( np.array([ convert_idx_to_one_hot(vocab, vocab.char_to_idx(SOS_TOKEN)) ])).unsqueeze(0) input = input.float().to(config.device) pred = '' for _ in range(8): out, hidden = model(input, hidden) if out.view(-1).div(0.8).exp().sum() == 0: continue topi = torch.multinomial(out.view(-1).div(0.8).exp(), 1)[0] if topi.item() == vocab.char_to_idx(EOS_TOKEN): break pred += vocab.idx_to_char(topi.item()) input = torch.from_numpy( np.array([convert_idx_to_one_hot(vocab, topi.item())])).unsqueeze(0) input = input.float().to(config.device) return pred
import numpy as np import torch from flask import Flask, request, jsonify, render_template from torch.nn import functional as F from model import CharRNN from settings import * from utils import load_dict, create_tune_header app = Flask(__name__) print("Environment:", app.config["ENV"]) # Create and load model model = CharRNN(n_char) model.load_state_dict(torch.load(default_model_path, map_location='cpu')) model.eval() # Load necessary dictionaries int2char = load_dict(int2char_path) char2int = load_dict(char2int_path) counter = Value("i", 0) error_message = "We created some tunes, but it seems like we can't create music from these melodies.Please try again!" print("Ready!") @app.route("/") def generate_song(): return render_template("index.html")
class Trainer(object): def __init__(self, args): self.args = args self.device = torch.device('cuda' if self.args.cuda else 'cpu') self.convert = None self.model = None self.optimizer = None self.criterion = self.get_loss self.meter = AverageValueMeter() self.train_loader = None self.get_data() self.get_model() self.get_optimizer() def get_data(self): self.convert = TextConverter(self.args.txt, max_vocab=self.args.max_vocab) dataset = TextDataset(self.args.txt, self.args.len, self.convert.text_to_arr) self.train_loader = DataLoader(dataset, self.args.batch_size, shuffle=True, num_workers=self.args.num_workers) def get_model(self): self.model = CharRNN(self.convert.vocab_size, self.args.embed_dim, self.args.hidden_size, self.args.num_layers, self.args.dropout, self.args.cuda).to(self.device) if self.args.cuda: cudnn.benchmark = True def get_optimizer(self): optimizer = torch.optim.Adam(self.model.parameters(), lr=self.args.lr) self.optimizer = ScheduledOptim(optimizer) @staticmethod def get_loss(score, label): return nn.CrossEntropyLoss()(score, label.view(-1)) def save_checkpoint(self, epoch): if (epoch + 1) % self.args.save_interval == 0: model_out_path = self.args.save_file + "epoch_{}_model.pth".format( epoch + 1) torch.save(self.model, model_out_path) print("Checkpoint saved to {}".format(model_out_path)) def save(self): model_out_path = self.args.save_file + "final_model.pth" torch.save(self.model, model_out_path) print("Final model saved to {}".format(model_out_path)) @staticmethod def pick_top_n(predictions, top_n=5): top_predict_prob, top_predict_label = torch.topk(predictions, top_n, 1) top_predict_prob /= torch.sum(top_predict_prob) top_predict_prob = top_predict_prob.squeeze(0).cpu().numpy() top_predict_label = top_predict_label.squeeze(0).cpu().numpy() c = np.random.choice(top_predict_label, size=1, p=top_predict_prob) return c def train(self): self.meter.reset() self.model.train() for x, y in tqdm(self.train_loader): y = y.long() x, y = x.to(self.device), y.to(self.device) # Forward. score, _ = self.model(x) loss = self.criterion(score, y) # Backward. self.optimizer.zero_grad() loss.backward() # Clip gradient. nn.utils.clip_grad_norm_(self.model.parameters(), 5) self.optimizer.step() self.meter.add(loss.item()) print('perplexity: {}'.format(np.exp(self.meter.value()[0]))) def test(self): self.model.eval() begin = np.array([i for i in self.args.begin]) begin = np.random.choice(begin, size=1) text_len = self.args.predict_len samples = [self.convert.word_to_int(c) for c in begin] input_txt = torch.LongTensor(samples)[None] input_txt = input_txt.to(self.device) _, init_state = self.model(input_txt) result = samples model_input = input_txt[:, -1][:, None] with torch.no_grad(): for i in range(text_len): out, init_state = self.model(model_input, init_state) prediction = self.pick_top_n(out.data) model_input = torch.LongTensor(prediction)[None].to( self.device) result.append(prediction[0]) print(self.convert.arr_to_text(result)) def predict(self): self.model.eval() samples = [self.convert.word_to_int(c) for c in self.args.begin] input_txt = torch.LongTensor(samples)[None].to(self.device) _, init_state = self.model(input_txt) result = samples model_input = input_txt[:, -1][:, None] with torch.no_grad(): for i in range(self.args.predict_len): out, init_state = self.model(model_input, init_state) prediction = self.pick_top_n(out.data) model_input = torch.LongTensor(prediction)[None].to( self.device) result.append(prediction[0]) print(self.convert.arr_to_text(result)) def run(self): for e in range(self.args.max_epoch): print('===> EPOCH: {}/{}'.format(e + 1, self.args.max_epoch)) self.train() self.test() self.save_checkpoint(e) self.save()
def train(opt, x_train, x_val, dictionary_len): ''' Training a network Arguments --------- net: CharRNN network data: training data to train the network (text) epochs: Number of epochs to train batch_size: Number of mini-sequences per mini-batch, aka batch size seq_length: Number of character steps per mini-batch lr: learning rate clip: gradient clipping val_frac: Fraction of data to hold out for validation print_every: Number of steps for printing training and validation loss ''' torch.manual_seed(0) np.random.seed(0) random.seed(0) # Declaring the hyperparameters batch_size = opt.batch_size seq_length = int(opt.seq_length) epochs = 50 if torch.cuda.is_available(): device = "cuda" torch.cuda.manual_seed_all(0) else: device = "cpu" print(device) date = datetime.now().strftime('%y%m%d%H%M%S') if opt.nologs: writer = SummaryWriter(log_dir=f'logs/nologs/') else: writer = SummaryWriter(log_dir=f'logs/logs_{date}/') y_train = get_labels_text_prediction(x_train) train_dataset = TextDataset(x_train, y_train, max_len=seq_length) if not opt.onlytrain: y_val = get_labels_text_prediction(x_val) val_dataset = TextDataset(x_val, y_val, max_len=seq_length) val_loader = DataLoader(dataset=val_dataset, pin_memory=device == 'cuda', batch_size=batch_size, shuffle=False) train_loader = DataLoader(dataset=train_dataset, pin_memory=device == 'cuda', batch_size=batch_size, shuffle=True) model_params = { 'dictionary_len': dictionary_len, 'dropout': opt.dropout, 'hidden_size': opt.hidden_size, 'layers': opt.layers, 'embedding_len': 32, 'device': device, 'lr': opt.lr } model = CharRNN(**model_params).to(device) print(model) # embed() # summary(model, input_size=(channels, H, W)) # summary(model, input_size=(dictionary_len, 28, 28)) optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) criterion = nn.CrossEntropyLoss() if opt.scheduler: scheduler = ReduceLROnPlateau(optimizer, 'min', cooldown=3, factor=0.5, patience=10) global_step = 0 for j in trange(epochs, desc='T raining LSTM...'): for i, (x, y) in enumerate(train_loader): if i == len(train_loader) - 1: print("FER PADDING - DE MOMENT NO VA") continue model.train() x = x.to(device) y = y.to(device) # state_h, state_c = model.zero_state(opt.batch_size) # # Transfer data to GPU # state_h = state_h.to(device) # state_c = state_c.to(device) # DELETE PAST GRADIENTS optimizer.zero_grad() # FORWARD PASS --> ultim state , (tots) [ state_h[-1] == pred ] pred, (state_h, state_c) = model(x) # pred, (state_h, state_c) = model(x, (state_h, state_c)) # CALCULATE LOSS # pred = pred.transpose(1, 2) pred2 = pred.view(-1, dictionary_len) y2 = y.view(-1) loss = criterion(pred2, y2) loss_value = loss.item() # BACKWARD PASS loss.backward() # MINIMIZE LOSS optimizer.step() global_step += 1 if i % 100 == 0: writer.add_scalar('train/loss', loss_value, global_step) print('[Training epoch {}: {}/{}] Loss: {}'.format( j, i, len(train_loader), loss_value)) if not opt.onlytrain: val_loss = [] for i, (x, y) in enumerate(val_loader): if i == len(val_loader) - 1: # print("FER PADDING - DE MOMENT NO VA") continue model.eval() x = x.to(device) y = y.to(device) # state_h, state_c = model.zero_state(opt.batch_size) # state_h = state_h.to(device) # state_c = state_c.to(device) # NO BACKPROPAGATION # FORWARD PASS # pred, (state_h, state_c) = model(x, (state_h, state_c)) pred, (state_h, state_c) = model(x) # CALCULATE LOSS # pred = pred.transpose(1, 2) # pred = [batch x 40 x diccionary_len] # y = [batch x 40] pred2 = pred.view(-1, dictionary_len) y2 = y.view(-1) loss = criterion(pred2, y2) # loss = criterion(pred, y) val_loss.append(loss.item()) if i % 50 == 0: print('[Validation epoch {}: {}/{}] Loss: {}'.format( j, i, len(val_loader), loss.item())) writer.add_scalar('val/loss', np.mean(val_loss), j) if opt.scheduler: scheduler.step(np.mean(val_loss)) writer.add_scalar("lr", optimizer.param_groups[0]["lr"], j) predicted_words = inference_prediction(model, device, 500) # output = pred[0].unsqueeze(0) # [1,diccionary_len, 40] # predicted_words = do_inference_test(output, model, device) print(predicted_words) writer.add_text('val/Generated_Samples', predicted_words, j) checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), } # if j % 5 == 0: os.makedirs("weights/{}".format(date), exist_ok=True) torch.save(checkpoint, "weights/{}/checkpoint_{}.pt".format(date, j))