def test(model, device, test_loader, returnAllScores=False): """ Evaluate a deep neural network model Args: model: pytorch model object device: cuda or cpu test_dataloader: test image dataloader returnAllScores: If true addtionally return all confidence scores and ground truth Returns: test loss and average precision. If returnAllScores = True, check Args """ #import cbowkai #from cbowkai import modelll, make_context_vector, idx_to_word, word_to_idx, nomprebab model.train(False) running_loss = 0 running_ap = 0 criterion = torch.nn.BCEWithLogitsLoss(reduction='sum') m = torch.nn.Sigmoid() if returnAllScores == True: all_scores = np.empty((0, 40), float) ground_scores = np.empty((0, 40), float) with torch.no_grad(): for data, target, senten in tqdm(test_loader): # print(data.size(), target.size()) target = dealit(target) data = data.to(device) bs, ncrops, c, h, w = data.size() output = model(data.view(-1, c, h, w)) output = output.view(bs, ncrops, -1).mean(1) loss = criterion(output, target) running_loss += loss # sum up batch loss running_ap += get_ap_score(torch.Tensor.cpu(target).detach().numpy(), torch.Tensor.cpu(m(output)).detach().numpy()) #print(m(output),target) if returnAllScores == True: all_scores = np.append(all_scores, torch.Tensor.cpu(m(output)).detach().numpy(), axis=0) ground_scores = np.append(ground_scores, torch.Tensor.cpu(target).detach().numpy(), axis=0) del data, target, output torch.cuda.empty_cache() num_samples = float(len(test_loader.dataset)) avg_test_loss = running_loss.item() / num_samples test_map = running_ap / num_samples print('test_loss: {:.4f}, test_avg_precision:{:.3f}'.format( avg_test_loss, test_map)) return avg_test_loss,test_map
def train_model(model, device, optimizer, scheduler, train_loader, valid_loader, save_dir, model_num, epochs, log_file): """ Train a deep neural network model Args: model: pytorch model object device: cuda or cpu optimizer: pytorch optimizer object scheduler: learning rate scheduler object that wraps the optimizer train_dataloader: training image dataloader valid_dataloader: validation image dataloader save_dir: Location to save model weights, plots and log_file epochs: number of training epochs log_file: text file instance to record training and validation history Returns: Training history and Validation history (loss and average precision) """ import cbowkai from cbowkai import modelll, make_context_vector, idx_to_word, word_to_idx, nomprebab tr_loss, tr_map = [], [] val_loss, val_map = [], [] best_val_map = 0.0 # Each epoch has a training and validation phase for epoch in range(epochs): print("-------Epoch {}----------".format(epoch + 1)) log_file.write("Epoch {} >>".format(epoch + 1)) for phase in ['train', 'valid']: running_loss = 0.0 running_ap = 0.0 criterion = torch.nn.BCEWithLogitsLoss(reduction='sum') m = torch.nn.Sigmoid() if phase == 'train': model.train(True) # Set model to training mode for data, target, senten in tqdm(train_loader): # print(data) target = dealit(target) data = data.to(device) sentenlist = [] for tpp in senten: tpi = tpp.split() context = [tpi[0], tpi[1]] context_vector = make_context_vector( context, word_to_idx) nll_prob = modelll(context_vector) changeprob = nomprebab(nll_prob) sentenlist.append(changeprob) outputtt = torch.Tensor.cuda(torch.Tensor(sentenlist)) # zero the parameter gradients optimizer.zero_grad() output = model(data) output = output * 0.9 + outputtt * 0.1 loss = criterion(output, target) # Get metrics here running_loss += loss running_ap += get_ap_score( torch.Tensor.cpu(target).detach().numpy(), torch.Tensor.cpu(m(output)).detach().numpy()) # Backpropagate the system the determine the gradients loss.backward() # Update the paramteres of the model optimizer.step() # clear variables del data, target, output gc.collect() torch.cuda.empty_cache() # print("loss = ", running_loss) num_samples = float(len(train_loader.dataset)) tr_loss_ = running_loss / num_samples tr_map_ = running_ap / num_samples print(running_loss, running_ap, num_samples) print('train_loss: {:.4f}, train_avg_precision:{:.3f}'.format( tr_loss_, tr_map_)) log_file.write( 'train_loss: {:.4f}, train_avg_precision:{:.3f}, '.format( tr_loss_, tr_map_)) # Append the values to global arrays tr_loss.append(tr_loss_), tr_map.append(tr_map_) else: model.train(False) # Set model to evaluate mode # torch.no_grad is for memory savings with torch.no_grad(): for data, target, senten in tqdm(valid_loader): target = dealit(target) data = data.to(device) sentenlist = [] for tpp in senten: tpi = tpp.split() context = [tpi[0], tpi[1]] context_vector = make_context_vector( context, word_to_idx) nll_prob = modelll(context_vector) changeprob = nomprebab(nll_prob) sentenlist.append(changeprob) outputtt = torch.Tensor.cuda(torch.Tensor(sentenlist)) # zero the parameter gradients output = model(data) output = output * 0.9 + outputtt * 0.1 loss = criterion(output, target) running_loss += loss # sum up batch loss running_ap += get_ap_score( torch.Tensor.cpu(target).detach().numpy(), torch.Tensor.cpu(m(output)).detach().numpy()) del data, target, output gc.collect() torch.cuda.empty_cache() num_samples = float(len(valid_loader.dataset)) val_loss_ = running_loss.item() / num_samples val_map_ = running_ap / num_samples # Append the values to global arrays val_loss.append(val_loss_), val_map.append(val_map_) print('val_loss: {:.4f}, val_avg_precision:{:.3f}'.format( val_loss_, val_map_)) log_file.write( 'val_loss: {:.4f}, val_avg_precision:{:.3f}\n'.format( val_loss_, val_map_)) # Save model using val_acc if val_map_ >= best_val_map: best_val_map = val_map_ log_file.write("saving best weights...\n") torch.save( model.state_dict(), os.path.join(save_dir, "model-{}.pth".format(model_num))) scheduler.step() return ([tr_loss, tr_map], [val_loss, val_map])
def test(model, device, test_loader, returnAllScores=False): """ Evaluate a deep neural network model Args: model: pytorch model object device: cuda or cpu test_dataloader: test image dataloader returnAllScores: If true addtionally return all confidence scores and ground truth Returns: test loss and average precision. If returnAllScores = True, check Args """ import cbowkai from cbowkai import modelll, make_context_vector, idx_to_word, word_to_idx, nomprebab model.train(False) running_loss = 0 running_ap = 0 criterion = torch.nn.BCEWithLogitsLoss(reduction='sum') m = torch.nn.Sigmoid() if returnAllScores == True: all_scores = np.empty((0, 40), float) ground_scores = np.empty((0, 40), float) with torch.no_grad(): for data, target, senten in tqdm(test_loader): # print(data.size(), target.size()) target = dealit(target) data = data.to(device) bs, ncrops, c, h, w = data.size() sentenlist = [] for tpp in senten: tpi = tpp.split() context = [tpi[0], tpi[1]] context_vector = make_context_vector(context, word_to_idx) nll_prob = modelll(context_vector) changeprob = nomprebab(nll_prob) sentenlist.append(changeprob) outputtt = torch.Tensor.cuda(torch.Tensor(sentenlist)) output = model(data.view(-1, c, h, w)) output = output.view(bs, ncrops, -1).mean(1) output = output * 0.9 + outputtt * 0.1 loss = criterion(output, target) running_loss += loss # sum up batch loss running_ap += get_ap_score( torch.Tensor.cpu(target).detach().numpy(), torch.Tensor.cpu(m(output)).detach().numpy()) #print(m(output),target) if returnAllScores == True: all_scores = np.append(all_scores, torch.Tensor.cpu( m(output)).detach().numpy(), axis=0) ground_scores = np.append( ground_scores, torch.Tensor.cpu(target).detach().numpy(), axis=0) del data, target, output gc.collect() torch.cuda.empty_cache() num_samples = float(len(test_loader.dataset)) avg_test_loss = running_loss.item() / num_samples test_map = running_ap / num_samples print('test_loss: {:.4f}, test_avg_precision:{:.3f}'.format( avg_test_loss, test_map)) f = open('test_result.txt', 'w') f.write('test loss:' + str(avg_test_loss) + ' ' + 'test_ap:' + str(test_map)) f.close() if returnAllScores == False: return avg_test_loss, running_ap return avg_test_loss, running_ap, all_scores, ground_scores
def train(model, device, train_loader, optimizer, epoch, lamdaValue, opt): # set model as training mode targetNet, sourceImagenet, sourcePlaces = model targetNet.train() if opt.isSource: if opt.multi_source: sourceImagenet = sourceImagenet.train() sourcePlaces = sourcePlaces.train() elif (not opt.multi_source) and opt.sourceKind == 'imagenet': sourceImagenet = sourceImagenet.train() elif (not opt.multi_source) and opt.sourceKind == 'places365': sourcePlaces = sourcePlaces.train() losses = AverageMeter() Targetscores = AverageMeter() targetloss = AverageMeter() auxiliaryimagenet = AverageMeter() auxiliaryplaces = AverageMeter() softmax = nn.Softmax() sigmoid = torch.nn.Sigmoid() upsample = nn.Upsample(scale_factor=7/3, mode='bilinear') N_count = 0 for batch_idx, (images, y) in enumerate(train_loader): if opt.dataset == 'voc': y = y.float() if opt.dataset == 'places365': y = y.view(-1, ) images, y = images.to(device), y.to(device) N_count+= images.size(0) optimizer.zero_grad() if opt.isSource: targetOutput, auxiliaryOutput1, auxiliaryOutput2 = targetNet(images) else: targetOutput = targetNet(images) upsample_dataset = ['cifar10', 'cifar100', 'stl10'] if opt.isSource: with torch.no_grad(): if opt.dataset in upsample_dataset: images = images.detach() images = upsample(images) prob1 = None prob2 = None if opt.multi_source: prob1 = sourceImagenet(images) prob2 = sourcePlaces(images) elif (not opt.multi_source) and opt.sourceKind == 'imagenet': prob1 = sourceImagenet(images) elif (not opt.multi_source) and opt.sourceKind == 'places365': prob2 = sourcePlaces(images) targetloss, auxiliary_imagenet, auxiliary_places, loss = loss_fn(targetOutput,auxiliaryOutput1, auxiliaryOutput2, prob1, prob2, y, opt) else: singleLabel = False if opt.dataset == 'voc' else True loss = model_fit(targetOutput, y, opt.classifier_loss_method, singleLabel=singleLabel, T=opt.T) losses.update(loss.item(), images.size()[0]) y_pred = torch.max(targetOutput, 1)[1] if opt.dataset == 'voc': step_score = get_ap_score(torch.Tensor.cpu(y).detach().numpy(), torch.Tensor.cpu(sigmoid(targetOutput)).detach().numpy()) else: step_score = accuracy(targetOutput.data, y.data, topk=(1,))[0] Targetscores.update(step_score,images.size()[0]) loss.backward() optimizer.step() if (batch_idx) % 10 == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}, Accu: {:.2f}%'.format( epoch, N_count, len(train_loader.dataset), 100. * (batch_idx + 1) / len(train_loader), losses.avg, Targetscores.avg)) return losses, Targetscores
def validation(model, device, optimizer, test_loader, lamdaValue, opt): targetNet, sourceImagenet, sourcePlaces = model targetNet.eval() if opt.isSource: if opt.multi_source: sourceImagenet = sourceImagenet.eval() sourcePlaces = sourcePlaces.eval() elif (not opt.multi_source) and opt.sourceKind == 'imagenet': sourceImagenet = sourceImagenet.eval() elif (not opt.multi_source) and opt.sourceKind == 'places365': sourcePlaces = sourcePlaces.eval() accs = AverageMeter() losses = AverageMeter() softmax = nn.Softmax() sigmoid = torch.nn.Sigmoid() upsample = nn.Upsample(scale_factor=7/3, mode='bilinear') with torch.no_grad(): for images, y in test_loader: # distribute data to device images, y = images.to(device), y.to(device) if opt.dataset == 'places365': y = y.view(-1, ) if opt.dataset == 'voc': y = y.float() if opt.isSource: targetOutput, auxiliaryOutput1, auxiliaryOutput2 = targetNet(images) else: targetOutput = targetNet(images) if opt.isSource: images = upsample(images) prob1 = None prob2 = None if opt.multi_source: prob1 = sourceImagenet(images) prob2 = sourcePlaces(images) elif (not opt.multi_source) and opt.sourceKind == 'imagenet': prob1 = sourceImagenet(images) elif (not opt.multi_source) and opt.sourceKind == 'places365': prob2 = sourcePlaces(images) targetloss, auxiliary_imagenet, auxiliary_places, loss = loss_fn(targetOutput,auxiliaryOutput1, auxiliaryOutput2, prob1, prob2, y, opt) else: singleLabel = False if opt.dataset == 'voc' else True loss = model_fit(targetOutput, y, opt.classifier_loss_method, singleLabel=singleLabel, T=opt.T) losses.update(loss.item(), images.size()[0]) if opt.dataset == 'voc': prec = get_ap_score(torch.Tensor.cpu(y).detach().numpy(), torch.Tensor.cpu(sigmoid(targetOutput)).detach().numpy()) else: prec = accuracy(targetOutput.data, y.data, topk=(1,))[0] accs.update(prec.item(), images.size()[0]) print('\nTest set ({:d} samples): Average loss: {:.4f}, Accuracy: {:.2f}%\n'.format(len(test_loader.dataset), losses.avg, accs.avg)) return losses, accs