class Solver: def __init__(self, params): # Build data handler self.data_handler = DataHandler() self.data_handler.loadData(params['input']) params['inp_dim'] = self.data_handler.getDataShape()[1] logging.info("=" * 41) # Build model self.model = SPINEModel(params) self.dtype = torch.FloatTensor use_cuda = torch.cuda.is_available() if use_cuda: self.model.cuda() self.dtype = torch.cuda.FloatTensor # Select optimizer optim_selected = params['optim'] LR = 0.1 if optim_selected == 'sgd': self.optimizer = torch.optim.SGD(self.model.parameters(), lr=LR) elif optim_selected == 'adam': self.optimizer = torch.optim.Adam(self.model.parameters(), lr=LR) logging.info("=" * 41) def train(self, params): num_epochs, batch_size = params['num_epochs'], params['batch_size'], optimizer = self.optimizer dtype = self.dtype STEP_DENOM = 5 scheduler = StepLR(optimizer, step_size=math.ceil(num_epochs / STEP_DENOM), gamma=0.3) for iteration in range(num_epochs): # lr adjusting scheduler.step() # start epoch self.data_handler.shuffleTrain() num_batches = self.data_handler.getNumberOfBatches(batch_size) epoch_losses = np.zeros(4) # rl, asl, psl, total for batch_idx in range(num_batches): optimizer.zero_grad() batch_x, batch_y = self.data_handler.getBatch( batch_idx, batch_size, params['noise_level'], params['denoising']) batch_x = Variable(torch.from_numpy(batch_x), requires_grad=False).type(dtype) batch_y = Variable(torch.from_numpy(batch_y), requires_grad=False).type(dtype) out, h, loss, loss_terms = self.model(batch_x, batch_y) reconstruction_loss, psl_loss, asl_loss = loss_terms loss.backward() epoch_losses[0] += reconstruction_loss.data.item() epoch_losses[1] += asl_loss.data.item() epoch_losses[2] += psl_loss.data.item() epoch_losses[3] += loss.data.item() optimizer.step() print("After epoch %r, Reconstruction Loss = %.4f, ASL = %.4f, " "PSL = %.4f, and total = %.4f" % (iteration + 1, epoch_losses[0], epoch_losses[1], epoch_losses[2], epoch_losses[3])) # logging.info("After epoch %r, Sparsity = %.1f" # %(iteration+1, utils.compute_sparsity(h.cpu().data.numpy()))) # break # break def getSpineEmbeddings(self, batch_size, params): ret = [] self.data_handler.resetDataOrder() num_batches = self.data_handler.getNumberOfBatches(batch_size) for batch_idx in range(num_batches): batch_x, batch_y = self.data_handler.getBatch( batch_idx, batch_size, params['noise_level'], params['denoising']) batch_x = Variable(torch.from_numpy(batch_x), requires_grad=False).type(self.dtype) batch_y = Variable(torch.from_numpy(batch_y), requires_grad=False).type(self.dtype) _, h, _, _ = self.model(batch_x, batch_y) ret.extend(h.cpu().data.numpy()) return np.array(ret) def getWordsList(self): return self.data_handler.getWordsList()
class Solver: def __init__(self, params): # Build data handler self.data_handler = DataHandler() self.data_handler.loadData(params['input']) params['inp_dim'] = self.data_handler.getDataShape()[1] logging.info("=" * 41) # Build model self.model = SPINEModel(params) self.dtype = torch.FloatTensor use_cuda = torch.cuda.is_available() if use_cuda: self.model.cuda() self.dtype = torch.cuda.FloatTensor self.optimizer = torch.optim.SGD(self.model.parameters(), lr=0.1) logging.info("=" * 41) def train(self, params): num_epochs, batch_size = params['num_epochs'], params['batch_size'], optimizer = self.optimizer dtype = self.dtype for iteration in range(num_epochs): self.data_handler.shuffleTrain() num_batches = self.data_handler.getNumberOfBatches(batch_size) epoch_losses = np.zeros(5) # rl, asl, psl, Siml, total sparsity = 0 for batch_idx in range(num_batches): optimizer.zero_grad() batch_x, batch_y = self.data_handler.getBatch( batch_idx, batch_size, params['noise_level'], params['denoising']) batch_x = Variable(torch.from_numpy(batch_x), requires_grad=False).type(dtype) batch_y = Variable(torch.from_numpy(batch_y), requires_grad=False).type(dtype) out, h, loss, loss_terms, sparsity_ratio = self.model( batch_x, batch_y) reconstruction_loss, psl_loss, asl_loss, siml = loss_terms loss.backward() optimizer.step() epoch_losses[0] += reconstruction_loss.data epoch_losses[1] += asl_loss.data epoch_losses[2] += psl_loss.data epoch_losses[3] += loss.data epoch_losses[4] += siml.data sparsity += sparsity_ratio.data print("After epoch %r, Reconstruction Loss = %.4f, ASL = %.4f,"\ "PSL = %.4f, SimL = %.4f , and total = %.4f, sparsity = %.4f" %(iteration+1, epoch_losses[0], epoch_losses[1], epoch_losses[2], epoch_losses[4], epoch_losses[3], sparsity/num_batches) ) if iteration % 1000 == 0: spine_embeddings = self.getSpineEmbeddings(512, params) utils.dump_vectors(spine_embeddings, './' + str(iteration / 1000) + '.txt', self.getWordsList()) #logging.info("After epoch %r, Sparsity = %.1f" # %(iteration+1, utils.compute_sparsity(h.cpu().data.numpy()))) #break #break def getSpineEmbeddings(self, batch_size, params): ret = [] self.data_handler.resetDataOrder() num_batches = self.data_handler.getNumberOfBatches(batch_size) for batch_idx in range(num_batches): batch_x, batch_y = self.data_handler.getBatch( batch_idx, batch_size, params['noise_level'], params['denoising']) batch_x = Variable(torch.from_numpy(batch_x), requires_grad=False).type(self.dtype) batch_y = Variable(torch.from_numpy(batch_y), requires_grad=False).type(self.dtype) _, h, _, _, spars = self.model(batch_x, batch_y) ret.extend(h.cpu().data.numpy()) return np.array(ret) def getWordsList(self): return self.data_handler.getWordsList()
class Solver: def __init__(self, params): # Build data handler self.data_handler = DataHandler() self.data_handler.loadData(params['input']) params['inp_dim'] = self.data_handler.getDataShape()[ 1] # inp_dim = 1000 logging.info("=" * 41) # Build model self.model = SPINEModel(params) self.dtype = torch.FloatTensor use_cuda = torch.cuda.is_available() if use_cuda: self.model.cuda() self.dtype = torch.cuda.FloatTensor self.optimizer = torch.optim.SGD(self.model.parameters(), lr=0.1) logging.info("=" * 41) def train(self, params): num_epochs, batch_size = params['num_epochs'], params['batch_size'] optimizer = self.optimizer dtype = self.dtype for iteration in range(num_epochs): # for each epoch self.data_handler.shuffleTrain() # 15000 training data shuffled num_batches = self.data_handler.getNumberOfBatches( batch_size) # num_batches = number of iterations epoch_losses = np.zeros( 5) # rl, asl, psl, total !!!!!!!!!!!!!!!!!! for batch_idx in range(num_batches): # for each iteration optimizer.zero_grad() batch_x, batch_y = self.data_handler.getBatch( batch_idx, batch_size, params['noise_level'], params['denoising']) batch_x = Variable(torch.from_numpy(batch_x), requires_grad=False).type(dtype) batch_y = Variable(torch.from_numpy(batch_y), requires_grad=False).type( dtype) #dtype = torch.FloatTensor out, h, loss, loss_terms = self.model(batch_x, batch_y) reconstruction_loss, psl_loss, asl_loss, local_loss = loss_terms #!!!!!!!!!!!!!!!!!!!!! loss.backward() optimizer.step() #print(local_loss) epoch_losses[0] += reconstruction_loss.item() epoch_losses[1] += asl_loss.item() epoch_losses[2] += psl_loss.item() epoch_losses[3] += local_loss.item() #!!!!!!!!!!!!!!!!!!!!!!!! epoch_losses[4] += loss.item() # epoch_losses[0]+=reconstruction_loss.data[0] # epoch_losses[1]+=asl_loss.data[0] # epoch_losses[2]+=psl_loss.data[0] # epoch_losses[3]+=local_loss.data[0] # epoch_losses[4]+=loss.data[0] print("After epoch %r, Reconstruction Loss = %.4f, ASL = %.4f,"\ "PSL = %.4f, Local Loss = %.4f and total = %.4f" %(iteration+1, epoch_losses[0], epoch_losses[1], epoch_losses[2], epoch_losses[3], epoch_losses[4]) ) #!!!!!! #logging.info("After epoch %r, Sparsity = %.1f" # %(iteration+1, utils.compute_sparsity(h.cpu().data.numpy()))) #break #break def getSpineEmbeddings(self, batch_size, params): ret = [] self.data_handler.resetDataOrder() num_batches = self.data_handler.getNumberOfBatches(batch_size) for batch_idx in range(num_batches): batch_x, batch_y = self.data_handler.getBatch( batch_idx, batch_size, params['noise_level'], params['denoising']) batch_x = Variable(torch.from_numpy(batch_x), requires_grad=False).type(self.dtype) batch_y = Variable(torch.from_numpy(batch_y), requires_grad=False).type(self.dtype) _, h, _, _ = self.model(batch_x, batch_y) ret.extend(h.cpu().data.numpy()) return np.array(ret) def getWordsList(self): return self.data_handler.getWordsList()
class Solver: def __init__(self, params): # build data handler self.data_handler = DataHandler() self.data_handler.loadData(params['input']) params['inp_dim'] = self.data_handler.getDataShape()[1] print("=" * 41) # build model self.model = SPINEModel(params) self.dtype = torch.FloatTensor # check if GPU is available use_cuda = torch.cuda.is_available() if use_cuda: self.model.cuda() self.dtype = torch.cuda.FloatTensor # set optimizer self.optimizer = torch.optim.SGD(self.model.parameters(), lr=0.1) print("=" * 41) def train(self, params): num_epochs, batch_size = params['num_epochs'], params['batch_size'], optimizer = self.optimizer dtype = self.dtype # train for each epoch for iteration in range(num_epochs): self.data_handler.shuffleTrain() num_batches = self.data_handler.getNumberOfBatches(batch_size) epoch_losses = np.zeros(4) # rl, asl, psl, total # for each batch for batch_idx in range(num_batches): optimizer.zero_grad() batch_x, batch_y = self.data_handler.getBatch( batch_idx, batch_size, params['noise_level'], params['denoising']) # transform batches into tensors batch_x = Variable(torch.from_numpy(batch_x), requires_grad=False).type(dtype) batch_y = Variable(torch.from_numpy(batch_y), requires_grad=False).type(dtype) # calculate losses out, h, loss, loss_terms = self.model(batch_x, batch_y) reconstruction_loss, psl_loss, asl_loss = loss_terms # update loss and optimizer loss.backward() optimizer.step() # assign loss epoch_losses[0] += reconstruction_loss.data epoch_losses[1] += asl_loss.data epoch_losses[2] += psl_loss.data epoch_losses[3] += loss.data print("epoch %r: Reconstruction Loss = %.4f, ASL = %.4f, "\ "PSL = %.4f, and total = %.4f" %(iteration+1, epoch_losses[0], epoch_losses[1], epoch_losses[2], epoch_losses[3]) ) def getSpineEmbeddings(self, batch_size, params): ret = [] self.data_handler.resetDataOrder() num_batches = self.data_handler.getNumberOfBatches(batch_size) # for each batch for batch_idx in range(num_batches): batch_x, batch_y = self.data_handler.getBatch( batch_idx, batch_size, params['noise_level'], params['denoising']) # transform batches into tensors batch_x = Variable(torch.from_numpy(batch_x), requires_grad=False).type(self.dtype) batch_y = Variable(torch.from_numpy(batch_y), requires_grad=False).type(self.dtype) _, h, _, _ = self.model(batch_x, batch_y) # append to embeddings ret.extend(h.cpu().data.numpy()) return np.array(ret) def getWordsList(self): return self.data_handler.getWordsList()