def __init__(self, hidden_size, output_size): super(DecoderRNN, self).__init__() self.hidden_size = hidden_size self.gru = nn.GRU(getTotalTokens(), hidden_size) self.out = nn.Linear(hidden_size, output_size) self.softmax = nn.LogSoftmax(dim=1)
def __init__(self, hidden_size, output_size): super(DecoderRNN, self).__init__() self.hidden_size = hidden_size self.gru = nn.GRU(getTotalTokens(), hidden_size) self.gru2 = nn.GRU(hidden_size, hidden_size) self.out = nn.Linear(hidden_size, output_size) self.m = nn.Sigmoid()
def trainSingleExample(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=100): encoder_hidden = encoder.initHidden() encoder_optimizer.zero_grad() decoder_optimizer.zero_grad() input_length = input_tensor.shape[0] # number of timesteps target_length = target_tensor.shape[0] encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device) loss = 0 for ei in range(input_length): encoder_output, encoder_hidden = encoder(input_tensor[ei,:], encoder_hidden) encoder_outputs[ei] = encoder_output[0, 0] startToken = np.zeros(getTotalTokens()) startToken[getStartIndex()] = 1 decoder_input = torch.tensor(startToken, device=device).float() decoder_hidden = encoder_hidden use_teacher_forcing = True if use_teacher_forcing: # Teacher forcing: Feed the target as the next input for di in range(target_length): decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden) #TODO: Use multi-label loss (Tied) ??? #todo: https://pytorch.org/docs/stable/nn.html#bceloss (or #bcewithlogitsloss) t = target_tensor[di,:] t = torch.argmax(t, dim=None) t = torch.Tensor([t]) loss += criterion(decoder_output, t.long()) decoder_input = target_tensor[di,:] # todo: is offset correct? #print(torch.argmax(decoder_input)) else: raise NotImplementedError loss.backward() encoder_optimizer.step() decoder_optimizer.step() return loss.item() / target_length
def trainSingleExample(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=100): encoder_hidden = encoder.initHidden() encoder_optimizer.zero_grad() decoder_optimizer.zero_grad() input_length = input_tensor.shape[0] # number of timesteps target_length = target_tensor.shape[0] encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device) loss = 0 for ei in range(input_length): encoder_output, encoder_hidden = encoder(input_tensor[ei,:], encoder_hidden) encoder_outputs[ei] = encoder_output[0,0] startToken = np.zeros(getTotalTokens()) startToken[getStartIndex()] = 1 decoder_input = torch.tensor(startToken, device=device).float() decoder_hidden = encoder_hidden use_teacher_forcing = True if use_teacher_forcing: # Teacher forcing: Feed the target as the next input for di in range(target_length): decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden) loss += criterion(decoder_output, target_tensor[di,:].view(1,-1)) decoder_input = target_tensor[di,:] # todo: is offset correct? else: raise NotImplementedError loss.backward() encoder_optimizer.step() decoder_optimizer.step() return loss.item() / target_length
decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate) piece, notes = entchen.get() encoderInput, decoderInput, decoderTarget = generateInput(notes, delta=1) criterion = nn.CrossEntropyLoss() input_tensor = torch.Tensor(encoderInput) target_tensor = torch.Tensor(decoderTarget) for i in range(0, epochs): loss = trainSingleExample(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion) print("Epoch", i+1, "finished. Loss:", loss) ########################## hidden_size = 256 encoder1 = EncoderRNN(getTotalTokens(), hidden_size).to(device) decoder1 = DecoderRNN(hidden_size, getTotalTokens()).to(device) train(encoder1, decoder1, epochs=100) torch.save(encoder1, "encoder.pt") torch.save(decoder1, "decoder.pt")
piece, notes = entchen.get() encoderInput, decoderInput, decoderTarget = generateInput(notes, delta=1, useTied=True, split=0.5) input = torch.Tensor(encoderInput) encoder = torch.load("encoder.pt") decoder = torch.load("decoder.pt") # Encoder Inference encoder_hidden = encoder.initHidden() for i in range(input.shape[0]): encoder_output, encoder_hidden = encoder(input[i, :], encoder_hidden) # Decoder Inference startToken = np.zeros(getTotalTokens()) startToken[getStartIndex()] = 1 decoder_input = torch.Tensor(startToken).float() decoder_hidden = encoder_hidden decoded_sentence = [] t = -1 while t != getStopIndex() and len(decoded_sentence) < MAX_LENGTH: decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden) tieProb = decoder_output[0,getTiedIndex()].item() tieProb = round(tieProb, 1) decoder_output[0,getTiedIndex()] = 0 t = torch.argmax(decoder_output)
from __future__ import unicode_literals, print_function, division import torch import torch.nn as nn from torch import optim from dataset import entchen from deprecated.pytorchVanillaRNN.VanillaLSTM import VanillaLSTM from deprecated.encodeNotes import getTotalTokens, getStopIndex, generateInput device = torch.device("cuda" if torch.cuda.is_available() else "cpu") ########################## hidden_size = 256 model = VanillaLSTM(getTotalTokens(), getTotalTokens()).to(device) piece, notes = entchen.get() encoderInput, decoderInput, decoderTarget = generateInput(notes, delta=1) input = torch.tensor(encoderInput).float() target = torch.tensor(decoderTarget).float() print(input.shape, target.shape) input = input.view(16, 1, 132) # seq.length,batch_size,dimensions print(input.shape, target.shape) output, hidden = model(input, model.initHidden()) print(output.shape) #print(hidden) optimizer = optim.SGD(model.parameters(), lr=0.01) #criterion = nn.BCELoss() criterion = nn.CrossEntropyLoss()