def test_update_parameters(): ''' (5 points) update_parameters''' m = LogisticRegression(3) # create a logistic regression object m.layer.weight.data = th.tensor([[0.5, 0.1, -0.2]]) m.layer.bias.data = th.tensor([0.2]) # create a toy loss function: the sum of all elements in w and b L = m.layer.weight.sum() + m.layer.bias.sum() # create an optimizer for w and b with learning rate = 0.1 optimizer = th.optim.SGD(m.parameters(), lr=0.1) # (step 1) back propagation to compute the gradients L.backward() assert np.allclose(m.layer.weight.grad, np.ones((3, 1)), atol=1e-2) assert np.allclose(m.layer.bias.grad, 1, atol=1e-2) # now perform gradient descent using SGD update_parameters(optimizer) # lets check the new values of the w and b assert np.allclose(m.layer.weight.data, [[0.4, 0., -0.3]], atol=1e-2) assert np.allclose(m.layer.bias.data, [0.1], atol=1e-2) # (step 2) back propagation again to compute the gradients L.backward() update_parameters(optimizer) assert np.allclose(m.layer.weight.data, [[0.3, -0.1, -0.4]], atol=1e-2) assert np.allclose(m.layer.bias.data, [0.], atol=1e-2) # (step 3) back propagation again to compute the gradients L.backward() update_parameters(optimizer) assert np.allclose(m.layer.weight.data, [[0.2, -0.2, -0.5]], atol=1e-2) assert np.allclose(m.layer.bias.data, [-0.1], atol=1e-2)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--dataset", type=str, default="usa", help="airport dataset to run experiment on") parser.add_argument("--embed_dim", type=int, default=128, help="struc2vec output embedding dimesnion") parser.add_argument("--epochs", type=int, default=10, help="number of epochs") parser.add_argument("--lr", type=float, default=0.01, help="learning rate") parser.add_argument("--l2", type=float, default=0.1, help="L2 regularization") args = parser.parse_args() dataset = args.dataset embed_dim = args.embed_dim epochs = args.epochs lr_rate = args.lr l2 = args.l2 num_classes = 4 test_accuracy = [] for i in range(10): print("Experiment {}".format(i)) model = LogisticRegression(embed_dim, num_classes) criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=lr_rate, weight_decay=l2) feat_data, labels, train_idx, test_idx = load_dataset( dataset, embed_dim) train(feat_data, labels, train_idx, model, criterion, optimizer, epochs) test_acc = evaluate(feat_data, labels, test_idx, model, criterion) test_accuracy.append(test_acc) print("Average performance: {}, standard deviation: {}".format( np.average(test_accuracy), np.std(test_accuracy)))
def train(data_loader, p, alpha=0.001, n_epoch=100): m = LogisticRegression(p) # initialize the model optimizer = th.optim.SGD(m.parameters(), lr=alpha) # create an SGD optimizer for _ in range(n_epoch): # iterate through the dataset n_epoch times for mini_batch in data_loader: # iterate through the dataset, with one mini-batch of random training samples (x,y) at a time x=mini_batch[0] # the feature vectors of the data samples in a mini-batch y=mini_batch[1] # the labels of the samples in a mini-batch ######################################### ## INSERT YOUR CODE HERE (5 points) L = compute_L(compute_z(x, m), y) L.backward() update_parameters(optimizer) ######################################### return m #----------------- '''
def main(): """ Train the model and print progress. """ data = gather_data() model = LogisticRegression(N_FEATS) optimizer = torch.optim.Adam(model.parameters()) for iteration in range(EPOCHS): print("Epoch {:d}".format(iteration + 1)) # Shuffle data. random.shuffle(data) total_loss = 0 running_loss = 0 n_examples = len(data) # Loop through examples in data. for i, example in enumerate(data): inp, tgt = example # Zero out the gradient. model.zero_grad() # Make a forward pass, i.e. compute the logits. logits = model(inp) loss = nn.functional.binary_cross_entropy_with_logits(logits, tgt) # Compute gradient and take step. loss.backward() optimizer.step() total_loss += loss running_loss += loss # Print progress. if (i + 1) % LOG_EVERY == 0: guess = logits[0].item() > 0 actual = tgt[0].item() == 1 correct = "✓" if guess == actual else "✗ ({:})".format(actual) print("({:d} / {:d}) Loss: {:.5f}".format( i + 1, n_examples, running_loss)) print(" => {:} {:}".format(guess, correct)) running_loss = 0 print("Epoch loss: {:f}\n".format(total_loss))
shuffle=False, num_workers=2) print(f"{datetime.now().ctime()} - Finish Loading Dataset") print( f"{datetime.now().ctime()} - Start Creating Net, Criterion, Optimizer and Scheduler..." ) conv_net = ConvNet(config.input_channel, 2) lr_model = LogisticRegression(config.cifar10_input_size) conv_criterion = nn.CrossEntropyLoss() lr_criterion = nn.BCEWithLogitsLoss() conv_optimizer = optim.SGD(conv_net.parameters(), config.lr, momentum=config.momentum, weight_decay=config.weight_decay) lr_optimizer = optim.SGD(lr_model.parameters(), config.lr, momentum=config.momentum, weight_decay=config.weight_decay) conv_scheduler = optim.lr_scheduler.CosineAnnealingLR(conv_optimizer, len(train_dataloader) * config.epochs, eta_min=config.eta_min) lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(lr_optimizer, len(train_dataloader) * config.epochs, eta_min=config.eta_min) print( f"{datetime.now().ctime()} - Finish Creating Net, Criterion, Optimizer and Scheduler" )
def train(train_dataset): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Using device:', device) train_loader = get_adni_loader(train_dataset, batch_size) model = LogisticRegression(input_size, num_classes) # Loss and Optimizer # Softmax is internally computed. # Set parameters to be updated. criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters()) # Training the Model loss_plot = [] net_dict = get_group_graphs(train_dataset) for epoch in range(num_epochs): for i, batch in enumerate(train_loader): nets = batch["network"] labels = batch["label"] thck = batch["thck"] n_node = thck.shape[1] n_edge = nets.shape[1] input = torch.cat((nets, thck), 1) input = Variable(input.view(-1, input_size)) labels = Variable(labels) # Forward + Backward + Optimize optimizer.zero_grad() outputs = model(input) loss = criterion(outputs, labels) sp_loss = 0 # print("Loss: ", loss) l1_crit = nn.L1Loss(size_average=False) reg_loss = 0 temp_loss = 0 l2_loss = nn.MSELoss(size_average=False) th_ls = 0 for param in model.parameters(): if len(param.shape) > 1: edge_param = param[:, :n_edge] node_param = param[:, n_edge:] reg_loss += l1_crit(param, torch.zeros_like(param)) #sp_loss += spatial_loss(edge_param, net_dict) for i in range(1, len(param)): temp_loss += l2_loss(param[i], param[i - 1]) #th_ls = thck_loss(edge_param, node_param) alpha_1 = torch.tensor(2.) alpha_2 = torch.tensor(0.) alpha_sp = torch.tensor(0.) alpha_thck = torch.tensor(0.) loss = loss + alpha_1 * reg_loss + alpha_2 * temp_loss + alpha_sp * sp_loss + \ alpha_thck * th_ls loss.backward() optimizer.step() if (epoch + 1) % 10 == 0: loss_plot.append(loss) print('Epoch: [%d/%d], Loss: %.4f' % (epoch + 1, num_epochs, loss.item())) # print("L1 Loss: {}\nSpatial Loss: {}\nTemporal Smoothness: {}\n".format(reg_loss, sp_loss, temp_loss)) plt.plot(loss_plot) plt.show() for param in model.linear.parameters(): print("Zeros: ", (param < 1e-9).sum()) return model