def main_run(version, train_data_dir, outDir, stackSize, trainBatchSize, valBatchSize, numEpochs, lr1, stepSize, decayRate, seqLen): num_classes = 61 model_folder = os.path.join("./", outDir, version) if os.path.exists(model_folder): print('Directory {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # Train val partitioning train_usr = ["S1", "S3", "S4"] val_usr = ["S2"] # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # imagenet mean and std spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) vid_seq_train = makeDataset(train_data_dir, train_usr, stackSize=stackSize, spatial_transform=spatial_transform, seqLen=seqLen) train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, sampler=None, num_workers=4, pin_memory=True) vid_seq_val = makeDataset(train_data_dir, val_usr, stackSize=stackSize, spatial_transform=Compose([ Scale(256), CenterCrop(224), ToTensor(), normalize ]), seqLen=seqLen) val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) valInstances = vid_seq_val.__len__() trainInstances = vid_seq_train.__len__() print('Number of samples in the dataset: training = {} | validation = {}'. format(trainInstances, valInstances)) model = flow_resnet34(True, channels=2 * stackSize, num_classes=num_classes) model.train(True) train_params = list(model.parameters()) model.cuda() loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.SGD(train_params, lr=lr1, momentum=0.9, weight_decay=5e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones=stepSize, gamma=decayRate) train_iter = 0 min_accuracy = 0 for epoch in range(numEpochs): start_time = time.time() optim_scheduler.step() epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 model.train(True) for i, (inputs, targets) in enumerate(train_loader): train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariable = Variable(inputs.to(DEVICE)) labelVariable = Variable(targets.to(DEVICE)) trainSamples += inputs.size(0) output_label, _ = model(inputVariable) loss = loss_fn(output_label, labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.to(DEVICE)).sum() epoch_loss += loss.data.item() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = (numCorrTrain.data.item() / trainSamples) * 100 print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format( epoch + 1, avg_loss, trainAccuracy)) train_log_loss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) print("--- time elapsed: %.2s seconds ---" % (time.time() - start_time)) if (epoch + 1) % 5 == 0: model.train(False) val_loss_epoch = 0 val_iter = 0 val_samples = 0 numCorr = 0 for j, (inputs, targets) in enumerate(val_loader): val_iter += 1 val_samples += inputs.size(0) inputVariable = Variable(inputs.to(DEVICE), volatile=True) labelVariable = Variable(targets.to(DEVICE)) output_label, _ = model(inputVariable) val_loss = loss_fn(output_label, labelVariable) val_loss_epoch += val_loss.data.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.to(DEVICE)).sum() val_accuracy = (numCorr.data.item() / val_samples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Validation: Epoch = {} | Loss = {} | Accuracy = {}'.format( epoch + 1, avg_val_loss, val_accuracy)) val_log_loss.write('Val Loss after {} epochs = {}\n'.format( epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format( epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_flow_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy # closing files train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close()
norm_method = Normalize(opt.mean, opt.std) if not opt.no_train: assert opt.train_crop in ['random', 'corner', 'center'] if opt.train_crop == 'random': crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size) elif opt.train_crop == 'corner': crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size) elif opt.train_crop == 'center': crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size, crop_positions=['c']) spatial_transform = Compose([ crop_method, RandomHorizontalFlip(opt.dataset), ToTensor(), norm_method ]) temporal_transform = TemporalSampling(opt.sample_duration) target_transform = ClassLabel() training_data = get_training_set(opt, spatial_transform, temporal_transform, target_transform) train_loader = torch.utils.data.DataLoader(training_data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.n_threads, pin_memory=True, drop_last=True) train_logger = Logger(os.path.join(opt.result_path, 'train.log'), ['epoch', 'loss', 'acc', 'lr']) train_batch_logger = Logger(
else: norm_method = Normalize(opt.mean, opt.std) if not opt.no_train: assert opt.train_crop in ['random', 'corner', 'center'] if opt.train_crop == 'random': crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size) elif opt.train_crop == 'corner': crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size) elif opt.train_crop == 'center': crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size, crop_positions=['c']) spatial_transform = Compose([ crop_method, RandomHorizontalFlip(), ToTensor(opt.norm_value), norm_method ]) temporal_transform = TemporalRandomCrop(opt.sample_duration) target_transform = ClassLabel() training_data = get_training_set(opt, spatial_transform, temporal_transform, target_transform) train_loader = torch.utils.data.DataLoader(training_data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.n_threads, pin_memory=True) train_logger = Logger(os.path.join(opt.result_path, 'train.log'), ['epoch', 'loss', 'acc', 'lr']) train_batch_logger = Logger( os.path.join(opt.result_path, 'train_batch.log'),
def main_run(dataset, trainDir, valDir, outDir, stackSize, trainBatchSize, valBatchSize, numEpochs, lr1, decay_factor, decay_step): if dataset == 'gtea61': num_classes = 61 elif dataset == 'gtea71': num_classes = 71 elif dataset == 'gtea_gaze': num_classes = 44 elif dataset == 'egtea': num_classes = 106 else: print('Dataset not found') sys.exit() min_accuracy = 0 model_folder = os.path.join('./', outDir, dataset, 'flow') # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Dir {} exists!'.format(model_folder)) !rm -rf ./experiments #sys.exit() os.makedirs(model_folder) # Log files writer = SummaryWriter(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize]) vid_seq_train = makeDataset(trainDir, spatial_transform=spatial_transform, sequence=False, stackSize=stackSize, fmt='.png') train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, sampler=None, num_workers=4, pin_memory=True) valInstances=0 if valDir is not None: vid_seq_val = makeDataset(valDir, spatial_transform=Compose([Scale(256), CenterCrop(224), ToTensor(), normalize]), sequence=False, stackSize=stackSize, fmt='.png', phase='Test') val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) valInstances = vid_seq_val.__len__() trainInstances = vid_seq_train.__len__() print('Number of samples in the dataset: training = {} | validation = {}'.format(trainInstances, valInstances)) model = flow_resnet34(True, channels=2*stackSize, num_classes=num_classes) model.train(True) train_params = list(model.parameters()) model.cuda() loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.SGD(train_params, lr=lr1, momentum=0.9, weight_decay=5e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 for epoch in range(numEpochs): optim_scheduler.step() epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 model.train(True) writer.add_scalar('lr', optimizer_fn.param_groups[0]['lr'], epoch+1) for i, (inputs, targets) in enumerate(train_loader): train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariable = Variable(inputs.cuda()) labelVariable = Variable(targets.cuda()) trainSamples += inputs.size(0) output_label, _ = model(inputVariable) loss = loss_fn(output_label, labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.cuda()).sum() epoch_loss += loss.data[0] avg_loss = epoch_loss/iterPerEpoch trainAccuracy = (numCorrTrain / trainSamples) * 100 print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format(epoch + 1, avg_loss, trainAccuracy)) writer.add_scalar('train/epoch_loss', avg_loss, epoch+1) writer.add_scalar('train/accuracy', trainAccuracy, epoch+1) train_log_loss.write('Training loss after {} epoch = {}\n'.format(epoch+1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format(epoch+1, trainAccuracy)) if valDir is not None: if (epoch+1) % 1 == 0: model.train(False) val_loss_epoch = 0 val_iter = 0 val_samples = 0 numCorr = 0 for j, (inputs, targets) in enumerate(val_loader): val_iter += 1 val_samples += inputs.size(0) inputVariable = Variable(inputs.cuda(), volatile=True) labelVariable = Variable(targets.cuda(async=True), volatile=True) output_label, _ = model(inputVariable) val_loss = loss_fn(output_label, labelVariable) val_loss_epoch += val_loss.data[0] _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.cuda()).sum() val_accuracy = (numCorr / val_samples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Validation: Epoch = {} | Loss = {} | Accuracy = {}'.format(epoch + 1, avg_val_loss, val_accuracy)) writer.add_scalar('val/epoch_loss', avg_val_loss, epoch + 1) writer.add_scalar('val/accuracy', val_accuracy, epoch + 1) val_log_loss.write('Val Loss after {} epochs = {}\n'.format(epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format(epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_flow_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy else: if (epoch+1) % 10 == 0: save_path_model = (model_folder + '/model_flow_state_dict_epoch' + str(epoch+1) + '.pth') torch.save(model.state_dict(), save_path_model) train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close() writer.export_scalars_to_json(model_folder + "/all_scalars.json") writer.close()
def main_run(dataset, stage, trainDatasetDir, valDatasetDir, stage1_dict, stackSize, out_dir, seqLen, trainBatchSize, valBatchSize, numEpochs, lr1, decay_factor, decay_step, memSize, alphaX, alphaY): if dataset == 'gtea61': num_classes = 61 elif dataset == 'gtea71': num_classes = 71 elif dataset == 'gtea_gaze': num_classes = 44 elif dataset == 'egtea': num_classes = 106 else: print('Dataset not found') sys.exit() model_folder = os.path.join( './', out_dir, 'attConvLSTM', str(seqLen), 'stage' + str(stage)) # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Directory {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files writer = SummaryWriter(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224) ]) spatial_transform2 = Compose([Scale((7, 7)), ToTensor()]) vid_seq_train = makeDataset(trainDatasetDir, spatial_transform2, spatial_transform=spatial_transform, sequence=False, numSeg=1, stackSize=stackSize, fmt='.png', seqLen=seqLen) trainInstances = vid_seq_train.__len__() train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=4, pin_memory=True) if valDatasetDir is not None: vid_seq_val = makeDataset(valDatasetDir, spatial_transform2, spatial_transform=Compose( [Scale(256), CenterCrop(224)]), sequence=False, numSeg=1, stackSize=stackSize, fmt='.png', phase='Test', seqLen=seqLen) valInstances = vid_seq_val.__len__() val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) train_params = [] if stage == 1: model = attentionModel(num_classes=num_classes, mem_size=memSize) model.train(False) for params in model.parameters(): params.requires_grad = False else: # stage == 2 model = attentionModel(num_classes=num_classes, mem_size=memSize) model.load_state_dict(torch.load(stage1_dict), strict=False) model.train(False) for params in model.parameters(): params.requires_grad = False # for params in model.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.fc.parameters(): params.requires_grad = True train_params += [params] model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) for params in model.lstm_cell.parameters(): params.requires_grad = True train_params += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] model.lstm_cell.train(True) model.classifier.train(True) model.cuda() loss_fn = nn.CrossEntropyLoss() loss_fn_regression = nn.MSELoss() # Loss function for the regression model optimizer_fn = torch.optim.Adam(train_params, lr=lr1, weight_decay=4e-5, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 min_accuracy = 0 for epoch in range(numEpochs): epoch_loss = 0 numCorrTrain = 0 x_loss = 0 y_loss = 0 trainSamples = 0 iterPerEpoch = 0 model.lstm_cell.train(True) model.classifier.train(True) writer.add_scalar('lr', optimizer_fn.param_groups[0]['lr'], epoch + 1) if stage == 2: model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) #for i, (inputs, targets) in enumerate(train_loader): for flowX, flowY, inputs, targets in train_loader: train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() flowX = flowX.cuda() flowY = flowY.cuda() inputVariable = Variable(inputs.permute(1, 0, 2, 3, 4).cuda()) labelVariable = Variable(targets.cuda()) trainSamples += inputs.size(0) output_label, _, flowXprediction, flowYprediction = model( inputVariable) #Reshaping predictions and inputs in order #to correctly regress on the inputs flowXprediction = flowXprediction.view(-1) flowX = torch.reshape(flowX, (-1, )).float() flowYprediction = flowYprediction.view(-1) flowY = torch.reshape(flowY, (-1, )).float() #print(f'Prediction: {flowXprediction.size()}') #print(f'Input : {flowX.size()}') #sys.exit() lossX = alphaX * loss_fn_regression(flowXprediction, flowX) lossY = alphaY * loss_fn_regression(flowYprediction, flowY) loss = loss_fn(output_label, labelVariable) #Weighting the loss of the ss task #by multiplying it by alpha total_loss = loss + lossX + lossY total_loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.cuda()).sum() x_loss += lossX.item() y_loss += lossY.item() epoch_loss += loss.item() optim_scheduler.step() avg_x_loss = x_loss / iterPerEpoch avg_y_loss = y_loss / iterPerEpoch avg_loss = epoch_loss / iterPerEpoch trainAccuracy = torch.true_divide(numCorrTrain, trainSamples) * 100 print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format( epoch + 1, avg_loss, trainAccuracy)) print('X loss after {} epoch = {}% '.format(epoch + 1, avg_x_loss)) print('Y loss after {} epoch = {}% '.format(epoch + 1, avg_y_loss)) writer.add_scalar('train/epoch_loss', avg_loss, epoch + 1) writer.add_scalar('train/accuracy', trainAccuracy, epoch + 1) writer.add_scalar('x_train_loss', avg_x_loss, epoch + 1) writer.add_scalar('y_train_loss', avg_y_loss, epoch + 1) train_log_loss.write('Training X loss after {} epoch= {}'.format( epoch + 1, avg_x_loss)) train_log_loss.write('Training Y loss after {} epoch= {}'.format( epoch + 1, avg_y_loss)) train_log_loss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) if valDatasetDir is not None: model.train(False) val_loss_epoch = 0 val_iter = 0 val_x_loss = 0 val_y_loss = 0 val_samples = 0 numCorr = 0 mmap_loss = 0 with torch.no_grad(): #for j, (inputs, targets) in enumerate(val_loader): for flowX, flowY, inputs, targets in val_loader: val_iter += 1 val_samples += inputs.size(0) flowX = flowX.cuda() flowY = flowY.cuda() inputVariable = Variable( inputs.permute(1, 0, 2, 3, 4).cuda()) labelVariable = Variable(targets.cuda(async=True)) #labelVariable = Variable(targets.cuda()) output_label, _, flowXprediction, flowYprediction = model( inputVariable) #Reshaping predictions and inputs in order #to correctly regress on the inputs flowXprediction = flowXprediction.view(-1) flowX = torch.reshape(flowX, (-1, )).float() flowYprediction = flowXprediction.view(-1) flowY = torch.reshape(flowX, (-1, )).float() lossX = alphaX * loss_fn_regression(flowXprediction, flowX) lossY = alphaY * loss_fn_regression(flowYprediction, flowY) val_loss = loss_fn(output_label, labelVariable) val_loss_epoch += val_loss.item() val_x_loss += lossX.item() val_y_loss += lossY.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.cuda()).sum() avg_x_val_loss = val_x_loss / val_iter avg_y_val_loss = val_y_loss / val_iter val_accuracy = torch.true_divide(numCorr, val_samples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Val X Loss after {} epochs, loss = {}'.format( epoch + 1, avg_x_val_loss)) print('Val Y Loss after {} epochs, loss = {}'.format( epoch + 1, avg_y_val_loss)) print('Val: Epoch = {} | Loss {} | Accuracy = {}'.format( epoch + 1, avg_val_loss, val_accuracy)) writer.add_scalar('val x/epoch_loss', avg_x_val_loss, epoch + 1) writer.add_scalar('val y/epoch_loss', avg_y_val_loss, epoch + 1) writer.add_scalar('val/epoch_loss', avg_val_loss, epoch + 1) writer.add_scalar('val/accuracy', val_accuracy, epoch + 1) val_log_loss.write('Val X Loss after {} epochs = {}\n'.format( epoch + 1, avg_x_val_loss)) val_log_loss.write('Val Y Loss after {} epochs = {}\n'.format( epoch + 1, avg_y_val_loss)) val_log_loss.write('Val Loss after {} epochs = {}\n'.format( epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format( epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_rgb_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close() writer.export_scalars_to_json(model_folder + "/all_scalars.json") writer.close()
def main_run(dataset, stage, root_dir, out_dir, stage1_dict, seqLen, trainBatchSize, numEpochs, lr1, decay_factor, decay_step, memSize, outPool_size, split, evalInterval, regression, rloss, debug): if debug: n_workers = 0 n_workers_test = 0 device = 'cpu' else: n_workers = 4 n_workers_test = 2 device = 'cuda' # Train/Validation/Test split train_splits = ["S1", "S3", "S4"] val_splits = ["S2"] test_split = split mean = [0.485, 0.456, 0.406] std = [0.229, 0.224, 0.225] normalize = Normalize(mean=mean, std=std) stage = stage #test_split = split seqLen = seqLen memSize = memSize c_cam_classes = outPool_size dataset = dataset best_acc = 0 if stage == 1: trainBatchSize = trainBatchSize testBatchSize = trainBatchSize lr1 = lr1 decay_factor = decay_factor decay_step = decay_step numEpochs = numEpochs elif stage == 2 or stage == 3: trainBatchSize = trainBatchSize testBatchSize = trainBatchSize lr1 = lr1 decay_factor = decay_factor decay_step = decay_step numEpochs = numEpochs num_classes = 61 dataset_dir = root_dir #model_folder = os.path.join('.', out_dir, dataset, str(test_split)) model_folder = os.path.join('./', out_dir, 'stage' + str(stage)) if not os.path.exists(model_folder): os.makedirs(model_folder) else: print('Directory {} exists!'.format(model_folder)) sys.exit() note_fl = open(model_folder + '/note.txt', 'w') note_fl.write('Number of Epochs = {}\n' 'lr = {}\n' 'Train Batch Size = {}\n' 'Sequence Length = {}\n' 'Decay steps = {}\n' 'Decay factor = {}\n' 'Memory size = {}\n' 'Memory cam classes = {}\n'.format(numEpochs, lr1, trainBatchSize, seqLen, decay_step, decay_factor, memSize, c_cam_classes)) note_fl.close() # Log files writer = SummaryWriter(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') train_log_loss_batch = open((model_folder + '/train_log_loss_batch.txt'), 'w') test_log_loss = open((model_folder + '/test_log_loss.txt'), 'w') test_log_acc = open((model_folder + '/test_log_acc.txt'), 'w') # Dataloaders spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) print('Preparing dataset...') # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), # ToTensor(), # normalize ]) transform_rgb = Compose([ToTensor(), normalize]) transform_MS = Compose([Resize((7, 7)), ToTensor()]) vid_seq_train = makeDataset(dataset_dir, splits=train_splits, spatial_transform=spatial_transform, transform_rgb=transform_rgb, transform_MS=transform_MS, seqLen=seqLen, fmt='.png', regression=regression) train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=n_workers, pin_memory=True) vid_seq_test = makeDataset(dataset_dir, splits=val_splits, spatial_transform=Compose( [Scale(256), CenterCrop(224)]), transform_rgb=transform_rgb, transform_MS=transform_MS, seqLen=seqLen, fmt='.png', regression=regression, verbose=False) test_loader = torch.utils.data.DataLoader(vid_seq_test, batch_size=testBatchSize, shuffle=False, num_workers=n_workers, pin_memory=True) print('here') print('Number of train samples = {}'.format(vid_seq_train.__len__())) print('Number of test samples = {}'.format(vid_seq_test.__len__())) train_params = [] if stage == 1: if regression: model = attentionModel(num_classes=num_classes, mem_size=memSize, n_channels=1) else: model = attentionModel(num_classes=num_classes, mem_size=memSize) model.train(False) for params in model.parameters(): params.requires_grad = False elif stage == 2 or stage == 3: if regression: model = attentionModel(num_classes=num_classes, mem_size=memSize, n_channels=1, c_cam_classes=c_cam_classes) else: model = attentionModel(num_classes=num_classes, mem_size=memSize, c_cam_classes=c_cam_classes) #model = attentionModel(num_classes=num_classes, mem_size=memSize, c_cam_classes=c_cam_classes) if stage == 2: checkpoint_path = os.path.join( stage1_dict, 'last_checkpoint_stage' + str(1) + '.pth.tar') elif stage == 3: checkpoint_path = os.path.join( stage1_dict, 'last_checkpoint_stage' + str(2) + '.pth.tar') if os.path.exists(checkpoint_path): print('Loading weights from checkpoint file {}'.format( checkpoint_path)) else: print('Checkpoint file {} does not exist'.format(checkpoint_path)) sys.exit() last_checkpoint = torch.load(checkpoint_path) model.load_state_dict(last_checkpoint['model_state_dict'], strict=False) model.train(False) for params in model.parameters(): params.requires_grad = False for params in model.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.fc.parameters(): params.requires_grad = True train_params += [params] # Add params from ms_module if stage == 2: for params in model.ms_module.parameters(): params.requires_grad = True train_params += [params] for params in model.lsta_cell.parameters(): params.requires_grad = True train_params += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] model.classifier.train(True) model.ms_module.train(True) model.to(device) loss_fn = nn.CrossEntropyLoss() if regression: if rloss == 'MSE': # Mean Squared Error loss loss_ms_fn = nn.MSELoss() # it should work elif rloss == 'L1': # L1 loss loss_ms_fn = nn.L1Loss() elif rloss == 'SmoothL1': # Huber Loss or Smooth L1 Loss loss_ms_fn = nn.SmoothL1Loss() elif rloss == 'KLdiv': # Kullback-Leiber Loss loss_ms_fn = nn.KLDivLoss() else: # classification loss_ms_fn = nn.CrossEntropyLoss() # TODO: check paper Planamente optimizer_fn = torch.optim.Adam(train_params, lr=lr1, weight_decay=5e-4, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 for epoch in range(numEpochs): #optim_scheduler.step() epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 # model.classifier.train(True) model.lsta_cell.train(True) model.classifier.train(True) if stage == 2: model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) model.ms_module.train(True) writer.add_scalar('lr', optimizer_fn.param_groups[0]['lr'], epoch + 1) for i, (inputs, inputsMS, targets) in enumerate(train_loader): # Inputs: # - inputsRGB : the rgb frame input # Labels : # - inputsMS : the motion task label # - targets : output train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariable = inputs.permute(1, 0, 2, 3, 4).to(device) labelVariable = targets.to(device) msVariable = inputsMS.to(device) trainSamples += inputs.size(0) output_label, _, output_ms = model(inputVariable, device) loss_c = loss_fn(output_label, labelVariable) if stage == 2: if regression: msVariable = torch.reshape( msVariable, (seqLen * 7 * 7, msVariable.size(0))) output_ms = torch.sigmoid(output_ms) output_ms = torch.reshape( output_ms, (seqLen * 7 * 7, output_ms.size(0))) else: # classification task msVariable = torch.reshape( msVariable, (seqLen * 7 * 7, msVariable.size(0))).long() output_ms = torch.reshape( output_ms, (seqLen * 7 * 7, 2, output_ms.size(0))) loss_ms = loss_ms_fn(output_ms, msVariable) loss = loss_c + loss_ms else: loss = loss_c loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.to(device)).sum() #print('Training loss after {} iterations = {} '.format(train_iter, loss.data.item())) #train_log_loss_batch.write('Training loss after {} iterations = {}\n'.format(train_iter, loss.data.item())) #writer.add_scalar('train/iter_loss', loss.data.item(), train_iter) epoch_loss += loss.data.item() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = (numCorrTrain / trainSamples) * 100 print('Average training loss after {} epoch = {} '.format( epoch + 1, avg_loss)) print('Training accuracy after {} epoch = {}% '.format( epoch + 1, trainAccuracy)) writer.add_scalar('train/epoch_loss', avg_loss, epoch + 1) writer.add_scalar('train/accuracy', trainAccuracy, epoch + 1) train_log_loss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) save_path_model = os.path.join( model_folder, 'last_checkpoint_stage' + str(stage) + '.pth.tar') save_file = { 'epoch': epoch + 1, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer_fn.state_dict(), 'best_acc': best_acc, } torch.save(save_file, save_path_model) if (epoch + 1) % evalInterval == 0: #print('Testing...') model.train(False) test_loss_epoch = 0 test_iter = 0 test_samples = 0 numCorr = 0 for j, (inputs, inputsMS, targets) in enumerate(test_loader): #print('testing inst = {}'.format(j)) test_iter += 1 test_samples += inputs.size(0) inputVariable = inputs.permute(1, 0, 2, 3, 4).to(device) labelVariable = targets.to(device) msVariable = inputsMS.to(device) output_label, _, output_ms = model(inputVariable, device) test_loss_c = loss_fn(output_label, labelVariable) if stage == 2: if regression: msVariable = torch.reshape( msVariable, (seqLen * 7 * 7, msVariable.size(0))) output_ms = torch.sigmoid(output_ms) output_ms = torch.reshape( output_ms, (seqLen * 7 * 7, output_ms.size(0))) else: # classification task msVariable = torch.reshape( msVariable, (seqLen * 7 * 7, msVariable.size(0))).long() output_ms = torch.reshape( output_ms, (seqLen * 7 * 7, 2, output_ms.size(0))) test_loss_ms = loss_ms_fn(output_ms, msVariable) test_loss = test_loss_c + test_loss_ms else: test_loss = test_loss_c test_loss_epoch += test_loss.data.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.to(device)).sum() test_accuracy = (numCorr / test_samples) * 100 avg_test_loss = test_loss_epoch / test_iter print('Test Loss after {} epochs, loss = {}'.format( epoch + 1, avg_test_loss)) print('Test Accuracy after {} epochs = {}%'.format( epoch + 1, test_accuracy)) writer.add_scalar('test/epoch_loss', avg_test_loss, epoch + 1) writer.add_scalar('test/accuracy', test_accuracy, epoch + 1) test_log_loss.write('Test Loss after {} epochs = {}\n'.format( epoch + 1, avg_test_loss)) test_log_acc.write('Test Accuracy after {} epochs = {}%\n'.format( epoch + 1, test_accuracy)) if test_accuracy > best_acc: best_acc = test_accuracy save_path_model = os.path.join( model_folder, 'best_checkpoint_stage' + str(stage) + '.pth.tar') save_file = { 'epoch': epoch + 1, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer_fn.state_dict(), 'best_acc': best_acc, } torch.save(save_file, save_path_model) optim_scheduler.step() train_log_loss.close() train_log_acc.close() test_log_acc.close() train_log_loss_batch.close() test_log_loss.close() writer.export_scalars_to_json(model_folder + "/all_scalars.json") writer.close()
def main_run(dataset, flowModel, rgbModel, stackSize, seqLen, memSize, trainDatasetDir, valDatasetDir, outDir, trainBatchSize, valBatchSize, lr1, numEpochs, decay_step, decay_factor, uniformSampling): # GTEA 61 num_classes = 61 # Train/Validation/Test split train_splits = ["S1", "S3", "S4"] val_splits = ["S2"] directory = trainDatasetDir model_folder = os.path.join( './', outDir, dataset, 'twoStream') # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Dir {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') mean = [0.485, 0.456, 0.406] std = [0.229, 0.224, 0.225] normalize = Normalize(mean=mean, std=std) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) vid_seq_train = makeDataset(directory, train_splits, spatial_transform=spatial_transform, sequence=False, numSeg=1, stackSize=stackSize, fmt='.png', seqLen=seqLen, uniform_sampling=uniformSampling) train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=4, pin_memory=True) vid_seq_val = makeDataset(directory, val_splits, spatial_transform=Compose([ Scale(256), CenterCrop(224), ToTensor(), normalize ]), sequence=False, numSeg=1, stackSize=stackSize, fmt='.png', phase='Test', seqLen=seqLen, uniform_sampling=uniformSampling) val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) valSamples = vid_seq_val.__len__() model = twoStreamAttentionModel(flowModel=flowModel, frameModel=rgbModel, stackSize=stackSize, memSize=memSize, num_classes=num_classes) for params in model.parameters(): params.requires_grad = False model.train(False) train_params = [] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.lstm_cell.parameters(): train_params += [params] params.requires_grad = True for params in model.frameModel.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.frameModel.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.frameModel.resNet.fc.parameters(): params.requires_grad = True train_params += [params] base_params = [] for params in model.flowModel.layer4.parameters(): base_params += [params] params.requires_grad = True model.cuda() trainSamples = vid_seq_train.__len__() min_accuracy = 0 loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.SGD([ { 'params': train_params }, { 'params': base_params, 'lr': 1e-4 }, ], lr=lr1, momentum=0.9, weight_decay=5e-4) optim_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_fn, step_size=decay_step, gamma=decay_factor) train_iter = 0 for epoch in range(numEpochs): epoch_loss = 0 numCorrTrain = 0 iterPerEpoch = 0 model.classifier.train(True) model.flowModel.layer4.train(True) for j, (inputFlow, inputFrame, targets) in enumerate(train_loader): train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariableFlow = inputFlow.to(DEVICE) inputVariableFrame = inputFrame.permute(1, 0, 2, 3, 4).to(DEVICE) labelVariable = targets.to(DEVICE) output_label = model(inputVariableFlow, inputVariableFrame) loss = loss_fn(torch.log_softmax(output_label, dim=1), labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.cuda()).sum() epoch_loss += loss.data.item() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = (numCorrTrain.item() / trainSamples) * 100 print('Average training loss after {} epoch = {} '.format( epoch + 1, avg_loss)) print('Training accuracy after {} epoch = {}% '.format( epoch + 1, trainAccuracy)) train_log_loss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) # Validation Phase #if valDatasetDir is not None: if (epoch + 1) % 1 == 0: model.train(False) val_loss_epoch = 0 val_iter = 0 numCorr = 0 for j, (inputFlow, inputFrame, targets) in enumerate(val_loader): val_iter += 1 inputVariableFlow = inputFlow.to(DEVICE) inputVariableFrame = inputFrame.permute(1, 0, 2, 3, 4).to(DEVICE) labelVariable = targets.to(DEVICE) output_label = model(inputVariableFlow, inputVariableFrame) loss = loss_fn(torch.log_softmax(output_label, dim=1), labelVariable) val_loss_epoch += loss.data.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == labelVariable.data).sum() val_accuracy = (numCorr.item() / valSamples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Val Loss after {} epochs, loss = {}'.format( epoch + 1, avg_val_loss)) print('Val Accuracy after {} epochs = {}%'.format( epoch + 1, val_accuracy)) val_log_loss.write('Val Loss after {} epochs = {}\n'.format( epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format( epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_twoStream_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy #else: # if (epoch + 1) % 10 == 0: # save_path_model = (model_folder + '/model_twoStream_state_dict_epoch' + str(epoch + 1) + '.pth') # torch.save(model.state_dict(), save_path_model) optim_scheduler.step() train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close()
def main_run(stage, train_data_dir, val_data_dir, stage1_dict, out_dir, seqLen, trainBatchSize, valBatchSize, numEpochs, lr1, decay_factor, decay_step, memSize, color, rgbm, fcm): #dataset = 'gtea61' begin_time = datetime.datetime.now() num_classes = 61 if color not in ['HSV_opticalFlow', 'flow_surfaceNormals', 'warpedHSV']: print(color, ' is not valid') exit(-1) model_folder = os.path.join( './', out_dir, 'BigConvLSTM', color, str(seqLen), 'stage' + str(stage)) # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Directory {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files writer = SummaryWriter(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) vid_seq_train = makeDataset(train_data_dir, seqLen=seqLen, fmt='.png', users=['S1', 'S3', 'S4'], spatial_transform=spatial_transform, colorization=color) #trainInstances = vid_seq_train.__len__() train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=4, pin_memory=True) if val_data_dir is not None: vid_seq_val = makeDataset(val_data_dir, seqLen=seqLen, fmt='.png', users=['S2'], train=False, spatial_transform=Compose([ Scale(256), CenterCrop(224), ToTensor(), normalize ]), colorization=color) #valInstances = vid_seq_val.__len__() val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) train_params = [] if stage == 1: model = bigConvLSTM(num_classes=num_classes, mem_size=memSize, rgbm=rgbm, fcm=fcm) model.train(False) for params in model.parameters(): params.requires_grad = False else: # stage == 2 model = bigConvLSTM(num_classes=num_classes, mem_size=memSize, rgbm=rgbm, fcm=fcm) model.load_state_dict(torch.load(stage1_dict)) model.train(False) for params in model.parameters(): params.requires_grad = False # for params in model.resNetRGB.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNetRGB.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNetRGB.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNetRGB.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNetRGB.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNetRGB.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNetRGB.fc.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNetCol.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNetCol.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNetCol.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNetCol.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNetCol.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNetCol.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNetCol.fc.parameters(): params.requires_grad = True train_params += [params] model.resNetRGB.layer4[0].conv1.train(True) model.resNetRGB.layer4[0].conv2.train(True) model.resNetRGB.layer4[1].conv1.train(True) model.resNetRGB.layer4[1].conv2.train(True) model.resNetRGB.layer4[2].conv1.train(True) model.resNetRGB.layer4[2].conv2.train(True) model.resNetRGB.fc.train(True) model.resNetCol.layer4[0].conv1.train(True) model.resNetCol.layer4[0].conv2.train(True) model.resNetCol.layer4[1].conv1.train(True) model.resNetCol.layer4[1].conv2.train(True) model.resNetCol.layer4[2].conv1.train(True) model.resNetCol.layer4[2].conv2.train(True) model.resNetCol.fc.train(True) for params in model.lstm_cell.parameters(): params.requires_grad = True train_params += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] model.lstm_cell.train(True) model.classifier.train(True) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) print(device) loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.Adam(train_params, lr=lr1, weight_decay=4e-5, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 min_accuracy = 0 dataload_time = datetime.datetime.now() for epoch in range(numEpochs): epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 model.lstm_cell.train(True) model.classifier.train(True) writer.add_scalar('lr', optimizer_fn.param_groups[0]['lr'], epoch + 1) if stage == 2: model.resNetRGB.layer4[0].conv1.train(True) model.resNetRGB.layer4[0].conv2.train(True) model.resNetRGB.layer4[1].conv1.train(True) model.resNetRGB.layer4[1].conv2.train(True) model.resNetRGB.layer4[2].conv1.train(True) model.resNetRGB.layer4[2].conv2.train(True) model.resNetRGB.fc.train(True) model.resNetCol.layer4[0].conv1.train(True) model.resNetCol.layer4[0].conv2.train(True) model.resNetCol.layer4[1].conv1.train(True) model.resNetCol.layer4[1].conv2.train(True) model.resNetCol.layer4[2].conv1.train(True) model.resNetCol.layer4[2].conv2.train(True) model.resNetCol.fc.train(True) #for i, (inputs, targets) in enumerate(train_loader): for inputsRGB, inputsCol, targets in train_loader: train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariableRGB = Variable( inputsRGB.permute(1, 0, 2, 3, 4).to(device)) inputVariableCol = Variable( inputsCol.permute(1, 0, 2, 3, 4).to(device)) labelVariable = Variable(targets.to(device)) trainSamples += inputsRGB.size(0) output_label, _ = model(inputVariableRGB, inputVariableCol, device) loss = loss_fn(output_label, labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.to(device)).sum() epoch_loss += loss.item() optim_scheduler.step() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = torch.true_divide(numCorrTrain, trainSamples) * 100 print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format( epoch + 1, avg_loss, trainAccuracy)) writer.add_scalar('train/epoch_loss', avg_loss, epoch + 1) writer.add_scalar('train/accuracy', trainAccuracy, epoch + 1) train_log_loss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) if val_data_dir is not None: model.train(False) val_loss_epoch = 0 val_iter = 0 val_samples = 0 numCorr = 0 with torch.no_grad(): #for j, (inputs, targets) in enumerate(val_loader): for inputsRGB, inputsCol, targets in val_loader: val_iter += 1 val_samples += inputsRGB.size(0) inputVariableRGB = Variable( inputsRGB.permute(1, 0, 2, 3, 4).to(device)) inputVariableCol = Variable( inputsCol.permute(1, 0, 2, 3, 4).to(device)) labelVariable = Variable(targets.to(device)) #labelVariable = Variable(targets.cuda()) output_label, _ = model(inputVariableRGB, inputVariableCol, device) val_loss = loss_fn(output_label, labelVariable) val_loss_epoch += val_loss.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.to(device)).sum() val_accuracy = torch.true_divide(numCorr, val_samples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Val: Epoch = {} | Loss {} | Accuracy = {}'.format( epoch + 1, avg_val_loss, val_accuracy)) writer.add_scalar('val/epoch_loss', avg_val_loss, epoch + 1) writer.add_scalar('val/accuracy', val_accuracy, epoch + 1) val_log_loss.write('Val Loss after {} epochs = {}\n'.format( epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format( epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_' + color + '_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy print("saved new best model") train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close() writer.export_scalars_to_json(model_folder + "/all_scalars.json") writer.close() end_time = datetime.datetime.now() print('total time elapsed: ', end_time - begin_time) print('dataload time: ', dataload_time - begin_time) print('training time: ', end_time - dataload_time) timers = open((model_folder + '/timings.txt'), 'w') timers.write( f"total time elapsed: {end_time-begin_time} \ndataload time: {dataload_time-begin_time} \ntraining time: {end_time-dataload_time}" ) timers.close()
def main_run(stage, model, supervision, train_data_dir, val_data_dir, stage1_dict, out_dir, seqLen, trainBatchSize, valBatchSize, numEpochs, lr1, lr_suphead, lr_resnet, alpha, decay_factor, decay_step, lossSupervision, memSize): num_classes = 61 if model == 'ConvLSTMAttention': model = ConvLSTMAttention(num_classes=num_classes, mem_size=memSize, supervision=supervision, loss_supervision=lossSupervision) elif model == 'ConvLSTM': model = ConvLSTM(num_classes=num_classes, mem_size=memSize, supervision=supervision, loss_supervision=lossSupervision) elif model == 'SupervisedLSTMMod': model = SupervisedLSTMMod(num_classes=num_classes, mem_size=memSize, supervision=supervision, loss_supervision=lossSupervision) elif model == 'MyNetIDT': model = MyNetIDT(num_classes=num_classes, mem_size=memSize, supervision=supervision, loss_supervision=lossSupervision) else: print('Model not found') sys.exit() model_folder = os.path.join( './', out_dir, 'rgb', 'stage' + str(stage)) # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Directory {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files writer = SummaryWriter(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) spatial_transform_map = Cp([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), Resize((7, 7)), TT() ]) spatial_transform_map_2 = Cp([Resize((7, 7)), TT()]) vid_seq_train = makeDataset_supervision( train_data_dir, train=True, spatial_transform=spatial_transform, spatial_transform_map=spatial_transform_map, seqLen=seqLen, fmt='.png') train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=8, pin_memory=True) if val_data_dir is not None: vid_seq_val = makeDataset_supervision( val_data_dir, train=False, spatial_transform_map=spatial_transform_map_2, spatial_transform=Compose( [Scale(256), CenterCrop(224), ToTensor(), normalize]), seqLen=seqLen, fmt='.png') val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=8, pin_memory=True) valInstances = vid_seq_val.__len__() trainInstances = vid_seq_train.__len__() train_params = [] train_params3 = [] train_params2 = [] if stage == 0: for params in model.resNet.parameters(): params.requires_grad = True train_params += [params] if stage1_dict is not None: model.load_state_dict(torch.load(stage1_dict)) elif stage == 1: supervision = False model.eval() for params in model.parameters(): params.requires_grad = False else: model.load_state_dict(torch.load(stage1_dict)) model.train() for params in model.parameters(): params.requires_grad = False # for params in model.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.fc.parameters(): params.requires_grad = True train_params += [params] model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) model.sup_head.train() for params in model.lstm_cell.parameters(): params.requires_grad = True train_params2 += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params2 += [params] for params in model.sup_head.parameters(): params.requires_grad = True train_params3 += [params] model.lstm_cell.train() model.classifier.train() model.cuda() if lossSupervision == "classification": loss_sup = nn.CrossEntropyLoss() elif lossSupervision == "regression": loss_sup = nn.L1Loss() loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.Adam([{ "params": train_params, "lr": lr_resnet }, { "params": train_params3, "lr": lr_suphead }, { "params": train_params2, "lr": lr1 }], lr=lr1, weight_decay=4e-5, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 min_accuracy = 0 for epoch in range(numEpochs): epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 epoch_loss_ = 0 loss_ = 0 model.lstm_cell.train(True) model.classifier.train(True) writer.add_scalar('lr', optimizer_fn.param_groups[0]['lr'], epoch + 1) if stage == 0: model.train() if stage == 2: model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.sup_head.train() model.resNet.fc.train(True) for i, (inputs, targets, maps) in enumerate(train_loader): train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() if lossSupervision == "classification": maps = torch.ceil(maps) maps = maps.type(torch.LongTensor) maps = maps.permute(1, 0, 2, 3, 4).squeeze(2).cuda() maps = maps.reshape(maps.shape[0] * maps.shape[1], maps.shape[2], maps.shape[3]) else: maps = maps.permute(1, 0, 2, 3, 4).cuda() maps = maps.reshape(maps.shape[0] * maps.shape[1], maps.shape[2], maps.shape[3], maps.shape[4]) inputVariable = Variable(inputs.permute(1, 0, 2, 3, 4).cuda()) labelVariable = Variable(targets.cuda()) trainSamples += inputs.size(0) output_label, _, output_super = model(inputVariable) if supervision == True: loss_ = alpha * loss_sup(output_super, maps) loss_.backward(retain_graph=True) epoch_loss_ += loss_.data.item() loss = loss_fn(output_label, labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.cuda()).sum() epoch_loss += loss.data.item() optim_scheduler.step() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = (numCorrTrain / float(trainSamples)) * 100 avg_loss_ = epoch_loss_ / float(iterPerEpoch) print( 'Train: Epoch = {} | Loss = {} | Accuracy = {} | supervision_loss {}' .format(epoch + 1, avg_loss, trainAccuracy, avg_loss_)) train_log_loss.write('Train Loss after {} epochs = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Train Accuracy after {} epochs = {}%\n'.format( epoch + 1, trainAccuracy)) writer.add_scalar('train/epoch_loss', avg_loss, epoch + 1) writer.add_scalar('train/accuracy', trainAccuracy, epoch + 1) if val_data_dir is not None: if (epoch + 1) % 1 == 0: model.eval() val_loss_epoch = 0 val_iter = 0 val_samples = 0 numCorr = 0 for j, (inputs, targets, _) in enumerate(val_loader): val_iter += 1 val_samples += inputs.size(0) with torch.no_grad(): inputVariable = Variable( inputs.permute(1, 0, 2, 3, 4).cuda()) labelVariable = Variable( targets.cuda(non_blocking=True)) output_label, _, _ = model(inputVariable) val_loss = loss_fn(output_label, labelVariable) val_loss_epoch += val_loss.data.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.cuda()).sum() val_accuracy = (numCorr / float(val_samples)) * 100 avg_val_loss = val_loss_epoch / val_iter print('val: Epoch = {} | Loss = {} | Accuracy = {} '.format( epoch + 1, avg_val_loss, val_accuracy)) writer.add_scalar('val/epoch_loss', avg_val_loss, epoch + 1) writer.add_scalar('val/accuracy', val_accuracy, epoch + 1) val_log_loss.write('Val Loss after {} epochs = {}\n'.format( epoch + 1, avg_val_loss)) val_log_acc.write( 'Val Accuracy after {} epochs = {}%\n'.format( epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_rgb_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy else: if (epoch + 1) % 10 == 0: save_path_model = (model_folder + '/model_rgb_state_dict_epoch' + str(epoch + 1) + '.pth') torch.save(model.state_dict(), save_path_model) train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close() writer.export_scalars_to_json(model_folder + "/all_scalars.json") writer.close()
def main(): opt = parse_opts() ecd_name, cls_name = opt.model_name.split('-') ecd_model = get_encoder_net(ecd_name) cls_model = get_end_net(cls_name) cfg.encoder_model = ecd_name cfg.classification_model = cls_name if opt.debug: cfg.debug = opt.debug else: if opt.tensorboard == 'TEST': cfg.tensorboard = opt.model_name else: cfg.tensorboard = opt.tensorboard cfg.flag = opt.flag model = cls_model(cfg, encoder=CNNencoder( cfg, ecd_model(pretrained=True, path=opt.encoder_model))) cfg.video_path = os.path.join(cfg.root_path, cfg.video_path) cfg.annotation_path = os.path.join(cfg.root_path, cfg.annotation_path) cfg.list_all_member() torch.manual_seed(cfg.manual_seed) print('##########################################') print('####### model 仅支持单GPU') print('##########################################') model = model.cuda() print(model) criterion = nn.CrossEntropyLoss() if cfg.cuda: criterion = criterion.cuda() norm_method = Normalize([0, 0, 0], [1, 1, 1]) print('##########################################') print('####### train') print('##########################################') assert cfg.train_crop in ['random', 'corner', 'center'] if cfg.train_crop == 'random': crop_method = (cfg.scales, cfg.sample_size) elif cfg.train_crop == 'corner': crop_method = MultiScaleCornerCrop(cfg.scales, cfg.sample_size) elif cfg.train_crop == 'center': crop_method = MultiScaleCornerCrop(cfg.scales, cfg.sample_size, crop_positions=['c']) spatial_transform = Compose([ crop_method, RandomHorizontalFlip(), ToTensor(cfg.norm_value), norm_method ]) temporal_transform = TemporalRandomCrop(cfg.sample_duration) target_transform = ClassLabel() training_data = get_training_set(cfg, spatial_transform, temporal_transform, target_transform) train_loader = torch.utils.data.DataLoader(training_data, batch_size=cfg.batch_size, shuffle=True, num_workers=cfg.n_threads, drop_last=False, pin_memory=True) optimizer = model.get_optimizer(lr1=cfg.lr, lr2=cfg.lr2) scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=cfg.lr_patience) print('##########################################') print('####### val') print('##########################################') spatial_transform = Compose([ Scale(cfg.sample_size), CenterCrop(cfg.sample_size), ToTensor(cfg.norm_value), norm_method ]) temporal_transform = LoopPadding(cfg.sample_duration) target_transform = ClassLabel() validation_data = get_validation_set(cfg, spatial_transform, temporal_transform, target_transform) val_loader = torch.utils.data.DataLoader(validation_data, batch_size=cfg.batch_size, shuffle=False, num_workers=cfg.n_threads, drop_last=False, pin_memory=True) print('##########################################') print('####### run') print('##########################################') if cfg.debug: logger = None else: path = get_log_dir(cfg.logdir, name=cfg.tensorboard, flag=cfg.flag) logger = Logger(logdir=path) cfg.save_config(path) for i in range(cfg.begin_epoch, cfg.n_epochs + 1): train_epoch(i, train_loader, model, criterion, optimizer, cfg, logger) validation_loss = val_epoch(i, val_loader, model, criterion, cfg, logger) scheduler.step(validation_loss)
def modelTrain(modelUsed, pretrained, trainDataset, trainLabels, validationDataset, validationLabels, numEpochs, evalInterval, evalMode, outDir, numWorkers, lr, stepSize, decayRate, trainBatchSize, seqLen, plotting): mean = [0.485, 0.456, 0.406] std = [0.229, 0.224, 0.225] normalize = Normalize(mean=mean, std=std) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) vidSeqTrain = makeDataset(trainDataset, trainLabels, spatial_transform=spatial_transform, seqLen=seqLen) # torch iterator to give data in batches of specified size trainLoader = torch.utils.data.DataLoader(vidSeqTrain, batch_size=trainBatchSize, shuffle=True, num_workers=numWorkers, pin_memory=True, drop_last=True) if evalMode == 'centerCrop': test_spatial_transform = Compose( [Scale(256), CenterCrop(224), ToTensor(), normalize]) elif evalMode == 'tenCrops': test_spatial_transform = Compose( [Scale(256), TenCrops(size=224, mean=mean, std=std)]) elif evalMode == 'fiveCrops': test_spatial_transform = Compose( [Scale(256), FiveCrops(size=224, mean=mean, std=std)]) elif evalMode == 'horFlip': test_spatial_transform = Compose([ Scale(256), CenterCrop(224), FlippedImagesTest(mean=mean, std=std) ]) vidSeqValid = makeDataset(validationDataset, validationLabels, seqLen=seqLen, spatial_transform=test_spatial_transform) validationLoader = torch.utils.data.DataLoader(vidSeqValid, batch_size=1, shuffle=False, num_workers=int(numWorkers / 2), pin_memory=True) numTrainInstances = vidSeqTrain.__len__() numValidationInstances = vidSeqValid.__len__() print('Number of training samples = {}'.format(numTrainInstances)) print('Number of validation samples = {}'.format(numValidationInstances)) modelFolder = './experiments_' + outDir + '_' + modelUsed + '_' + str( pretrained) # Dir for saving models and log files # Create the dir if os.path.exists(modelFolder): pass else: os.makedirs(modelFolder) # Log files writer = SummaryWriter(modelFolder) trainLogLoss = open((modelFolder + '/trainLogLoss.txt'), 'a') trainLogAcc = open((modelFolder + '/trainLogAcc.txt'), 'a') validationLogLoss = open((modelFolder + '/validLogLoss.txt'), 'a') validationLogAcc = open((modelFolder + '/validLogAcc.txt'), 'a') model = ViolenceModel(modelUsed, pretrained) trainParams = [] for params in model.parameters(): if params.requires_grad: trainParams += [params] model.train(True) if (torch.cuda.is_available()): model.cuda() lossFn = nn.CrossEntropyLoss() optimizerFn = torch.optim.RMSprop(trainParams, lr=lr) optimizerFn.zero_grad() optimScheduler = torch.optim.lr_scheduler.StepLR(optimizerFn, stepSize, decayRate) minAccuracy = 50 train_loss = [] val_loss = [] train_acc = [] val_acc = [] bestmodel = None for epoch in range(numEpochs): optimScheduler.step() epochLoss = 0 numCorrTrain = 0 iterPerEpoch = 0 model.train(True) print('Epoch = {}'.format(epoch + 1)) writer.add_scalar('lr', optimizerFn.param_groups[0]['lr'], epoch + 1) for i, (inputs, targets) in enumerate(trainLoader): iterPerEpoch += 1 optimizerFn.zero_grad() if (torch.cuda.is_available()): inputVariable1 = Variable(inputs.permute(1, 0, 2, 3, 4).cuda()) labelVariable = Variable(targets.cuda()) else: inputVariable1 = Variable(inputs.permute(1, 0, 2, 3, 4)) labelVariable = Variable(targets) outputLabel = model(inputVariable1) loss = lossFn(outputLabel, labelVariable) loss.backward() optimizerFn.step() outputProb = torch.nn.Softmax(dim=1)(outputLabel) _, predicted = torch.max(outputProb.data, 1) if (torch.cuda.is_available()): numCorrTrain += (predicted == targets.cuda()).sum() else: numCorrTrain += (predicted == targets).sum() epochLoss += loss.item() avgLoss = epochLoss / iterPerEpoch trainAccuracy = (float(numCorrTrain) * 100) / float(numTrainInstances) train_loss.append(avgLoss) train_acc.append(trainAccuracy) print('Training: Loss = {} | Accuracy = {}% '.format( avgLoss, trainAccuracy)) writer.add_scalar('train/epochLoss', avgLoss, epoch + 1) writer.add_scalar('train/accuracy', trainAccuracy, epoch + 1) trainLogLoss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avgLoss)) trainLogAcc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) if (epoch + 1) % evalInterval == 0: model.train(False) print('Evaluating...') validationLossEpoch = 0 validationIter = 0 numCorrTest = 0 for j, (inputs, targets) in enumerate(validationLoader): validationIter += 1 #if evalMode == 'centerCrop': if (torch.cuda.is_available()): inputVariable1 = Variable(inputs.permute(1, 0, 2, 3, 4).cuda(), requires_grad=False) labelVariable = Variable(targets.cuda(async=True), requires_grad=False) else: inputVariable1 = Variable(inputs.permute(1, 0, 2, 3, 4), requires_grad=False) labelVariable = Variable(targets, requires_grad=False) # else: # if(torch.cuda.is_available()): # inputVariable1 = Variable(inputs[0].permute(1, 0, 2, 3, 4).cuda(), requires_grad=False) # labelVariable = Variable(targets.cuda(async=True), requires_grad=False) # else: # inputVariable1 = Variable(inputs[0].permute(1, 0, 2, 3, 4), requires_grad=False) # labelVariable = Variable(targets, requires_grad=False) outputLabel = model(inputVariable1) validationLoss = lossFn(outputLabel, labelVariable) validationLossEpoch += validationLoss.item() outputProb = torch.nn.Softmax(dim=1)(outputLabel) _, predicted = torch.max(outputProb.data, 1) if (torch.cuda.is_available()): numCorrTest += (predicted == targets[0].cuda()).sum() else: numCorrTest += (predicted == targets[0]).sum() validationAccuracy = (float(numCorrTest) * 100) / float(numValidationInstances) avgValidationLoss = validationLossEpoch / validationIter val_loss.append(avgValidationLoss) val_acc.append(validationAccuracy) print('Testing: Loss = {} | Accuracy = {}% '.format( avgValidationLoss, validationAccuracy)) writer.add_scalar('test/epochloss', avgValidationLoss, epoch + 1) writer.add_scalar('test/accuracy', validationAccuracy, epoch + 1) validationLogLoss.write('valid Loss after {} epochs = {}\n'.format( epoch + 1, avgValidationLoss)) validationLogAcc.write( 'valid Accuracy after {} epochs = {}%\n'.format( epoch + 1, validationAccuracy)) if validationAccuracy > minAccuracy: bestmodel = model minAccuracy = validationAccuracy '''plotting the accuracy and loss curves''' if plotting: xc = range(1, numEpochs + 1) xv = [] for i in xc: if (i % evalInterval == 0): xv.append(i) plt.figure(1, figsize=(7, 5)) plt.plot(xc, train_loss) plt.plot(xv, val_loss) plt.xlabel('num of Epochs') plt.ylabel('loss') plt.title('train_loss vs val_loss') plt.grid(True) plt.legend(['train', 'val']) #print plt.style.available # use bmh, classic,ggplot for big pictures plt.style.use(['classic']) plt.savefig(modelFolder + "/lossCurve.png") plt.figure(2, figsize=(7, 5)) plt.plot(xc, train_acc) plt.plot(xv, val_acc) plt.xlabel('num of Epochs') plt.ylabel('accuracy') plt.title('train_acc vs val_acc') plt.grid(True) plt.legend(['train', 'val'], loc=4) #print plt.style.available # use bmh, classic,ggplot for big pictures plt.style.use(['classic']) plt.savefig(modelFolder + "/accuracyCurve.png") #plt.show() trainLogAcc.close() validationLogAcc.close() trainLogLoss.close() validationLogLoss.close() writer.export_scalars_to_json(modelFolder + "/all_scalars.json") writer.close() return bestmodel, validationAccuracy
def create_dataloader(args): if args.root_path != '': args.video_path = os.path.join(args.root_path, args.video_path) args.annotation_path = os.path.join(args.root_path, args.annotation_path) args.result_path = os.path.join(args.root_path, args.result_path) if args.resume_path: args.resume_path = os.path.join(args.root_path, args.resume_path) if args.pretrain_path: # args.pretrain_path = os.path.join(args.root_path, args.pretrain_path) args.pretrain_path = os.path.abspath(args.pretrain_path) args.scales = [args.initial_scale] for i in range(1, args.n_scales): args.scales.append(args.scales[-1] * args.scale_step) args.mean = get_mean(args.norm_value, dataset=args.mean_dataset) args.std = get_std(args.norm_value) if args.no_mean_norm and not args.std_norm: norm_method = Normalize([0, 0, 0], [1, 1, 1]) elif not args.std_norm: norm_method = Normalize(args.mean, [1, 1, 1]) else: norm_method = Normalize(args.mean, args.std) assert args.train_crop in ['random', 'corner', 'center'] if args.train_crop == 'random': crop_method = MultiScaleRandomCrop(args.scales, args.sample_size) elif args.train_crop == 'corner': crop_method = MultiScaleCornerCrop(args.scales, args.sample_size) elif args.train_crop == 'center': crop_method = MultiScaleCornerCrop(args.scales, args.sample_size, crop_positions=['c']) spatial_transform = Compose([ crop_method, RandomHorizontalFlip(), ToTensor(args.norm_value), norm_method ]) temporal_transform = TemporalRandomCrop(args.sample_duration) target_transform = ClassLabel() training_data = get_training_set(args, spatial_transform, temporal_transform, target_transform) train_loader = torch.utils.data.DataLoader(training_data, batch_size=args.batch_size, shuffle=True, num_workers=args.n_threads, pin_memory=True) spatial_transform = Compose([ # Scale(args.sample_size), Scale(int(args.sample_size / args.scale_in_test)), # CenterCrop(args.sample_size), CornerCrop(args.sample_size, args.crop_position_in_test), ToTensor(args.norm_value), norm_method ]) temporal_transform = TemporalCenterCrop(args.sample_duration) target_transform = ClassLabel() validation_data = get_validation_set(args, spatial_transform, temporal_transform, target_transform) val_loader = torch.utils.data.DataLoader(validation_data, batch_size=1, shuffle=False, num_workers=args.n_threads, pin_memory=True) return train_loader, val_loader
def main_run(stage, model, supervision, train_data_dir, val_data_dir, stage1dict, out_dir, seq_len, train_batch_size, val_batch_size, num_epochs, lr1, lr_suphead, lr_resnet, alpha, decay_factor, decay_step, mem_size): num_classes = 61 if model == 'MyNet': model = MyNet(num_classes=num_classes, mem_size=mem_size) elif model == 'ConvLSTMDynamic': model = ConvLSTMDynamic(num_classes=num_classes, mem_size=mem_size) else: print('Model not found') sys.exit() model_folder = os.path.join('./', out_dir, 'rgb') # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Directory {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files writer = SummaryWriter(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) unnormalize = UnNormalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize]) vid_seq_train = MakeDataset_flowsupervision(train_data_dir, train=True, spatial_transform=spatial_transform, seq_len=seq_len, fmt='.png') train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=train_batch_size, shuffle=True, num_workers=2, pin_memory=True) if val_data_dir is not None: vid_seq_val = MakeDataset_flowsupervision(val_data_dir, train=False, spatial_transform=Compose([Scale(256), CenterCrop(224), ToTensor(), normalize]), seq_len=seq_len, fmt='.png') val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=val_batch_size, shuffle=False, num_workers=2, pin_memory=True) train_params = [] train_params3 = [] train_params2 = [] if stage == 1: supervision = False model.eval() for params in model.parameters(): params.requires_grad = False else: model.load_state_dict(torch.load(stage1dict),strict=False) model.train() for params in model.parameters(): params.requires_grad = False # for params in model.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.fc.parameters(): params.requires_grad = True train_params += [params] model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) model.dinam.train() for params in model.lstm_cell.parameters(): params.requires_grad = True train_params2 += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params2 += [params] for params in model.dinam.parameters(): params.requires_grad = True train_params3 += [params] model.train() model.cuda() loss_sup = nn.MSELoss() loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.Adam([{"params": train_params, "lr": lr_resnet}, {"params": train_params3, "lr": lr_suphead}, {"params": train_params2, "lr": lr1}], lr=lr1, weight_decay=4e-5, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 min_accuracy = 0 for epoch in range(num_epochs): epoch_loss = 0 num_corr_train = 0 train_samples = 0 iter_per_epoch = 0 epoch_loss_ = 0 model.lstm_cell.train() model.classifier.train() if stage == 2: model.resNet.layer4[0].conv1.train() model.resNet.layer4[0].conv2.train() model.resNet.layer4[1].conv1.train() model.resNet.layer4[1].conv2.train() model.resNet.layer4[2].conv1.train() model.resNet.layer4[2].conv2.train() model.dinam.train() model.resNet.fc.train() writer.add_scalar('lr', optimizer_fn.param_groups[0]['lr'], epoch+1) for i, (inputs, targets, m) in enumerate(train_loader): train_iter += 1 iter_per_epoch += 1 optimizer_fn.zero_grad() images = inputs.permute(1, 0, 2, 3, 4).cuda() labels = targets.cuda() m = m.permute(1, 0, 2, 3, 4).cuda() train_samples += inputs.size(0) output_label, _, output_super = model(images) if supervision: loss_ = loss_sup(output_super, m.cuda()) epoch_loss_ += loss_.data.item() loss = loss_fn(output_label, labels) epoch_loss += loss.data.item() if supervision: loss = loss + loss_*alpha loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) num_corr_train += (predicted == targets.cuda()).sum() optim_scheduler.step() avg_loss = epoch_loss/iter_per_epoch train_acc = (num_corr_train / float(train_samples)) * 100 output_super = output_super.cpu() m = m.cpu() if (epoch % 2 == 0) and supervision: d = f"./Results/epoch{epoch}" os.makedirs(d) for i in range(m.shape[0]): save_image(unnormalize(m[i, 0]), d+f'/inp{i}.png') save_image(unnormalize(output_super[i, 0].detach()), d+f'/out{i}.png') avg_loss_ = epoch_loss_/float(iter_per_epoch) print('Train: Epoch = {} | Loss = {} | Accuracy = {} | supervision_loss {}'.format(epoch+1, avg_loss, train_acc, avg_loss_)) train_log_loss.write('Train Loss after {} epochs = {}\n'.format(epoch + 1, avg_loss)) train_log_acc.write('Train Accuracy after {} epochs = {}%\n'.format(epoch + 1, train_acc)) writer.add_scalar('train/epoch_loss', avg_loss, epoch+1) writer.add_scalar('train/accuracy', train_acc, epoch+1) if val_data_dir is not None: if (epoch+1) % 1 == 0: model.eval() val_loss_epoch = 0 val_iter = 0 val_samples = 0 num_corr = 0 for j, (inputs, targets, _) in enumerate(val_loader): val_iter += 1 val_samples += inputs.size(0) with torch.no_grad(): images = inputs.permute(1, 0, 2, 3, 4).cuda() labels = targets.cuda(non_blocking=True) output_label, _, _ = model(images) val_loss = loss_fn(output_label, labels) val_loss_epoch += val_loss.data.item() _, predicted = torch.max(output_label.data, 1) num_corr += (predicted == targets.cuda()).sum() val_accuracy = (num_corr / float(val_samples)) * 100 avg_val_loss = val_loss_epoch / val_iter print('val: Epoch = {} | Loss = {} | Accuracy = {} '.format(epoch+1, avg_val_loss, val_accuracy)) writer.add_scalar('val/epoch_loss', avg_val_loss, epoch + 1) writer.add_scalar('val/accuracy', val_accuracy, epoch + 1) val_log_loss.write('Val Loss after {} epochs = {}\n'.format(epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format(epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_rgb_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy else: if (epoch+1) % 10 == 0: save_path_model = (model_folder + '/model_rgb_state_dict_epoch' + str(epoch+1) + '.pth') torch.save(model.state_dict(), save_path_model) train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close() writer.export_scalars_to_json(model_folder + "/all_scalars.json") writer.close()
def main_run(dataset, stage, train_data_dir, val_data_dir, stage1_dict, out_dir, seqLen, trainBatchSize, valBatchSize, numEpochs, lr1, decay_factor, decay_step, memSize, CAM=True, debug=False): # GTEA 61 num_classes = 61 if debug: n_workers = 0 device = 'cpu' else: n_workers = 4 device = 'cuda' # Train/Validation/Test split train_splits = ["S1", "S3", "S4"] val_splits = ["S2"] model_folder = os.path.join( './', out_dir, dataset, 'rgb', 'stage' + str(stage)) # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Directory {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) vid_seq_train = makeDataset(train_data_dir, splits=train_splits, spatial_transform=spatial_transform, seqLen=seqLen, fmt='.png') train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=n_workers, pin_memory=True) vid_seq_val = makeDataset(train_data_dir, splits=val_splits, spatial_transform=Compose([ Scale(256), CenterCrop(224), ToTensor(), normalize ]), seqLen=seqLen, fmt='.png', verbose=False) val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=n_workers, pin_memory=True) valInstances = vid_seq_val.__len__() ''' if val_data_dir is not None: vid_seq_val = makeDataset(val_data_dir, spatial_transform=Compose([Scale(256), CenterCrop(224), ToTensor(), normalize]), seqLen=seqLen, fmt='.jpg') val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) valInstances = vid_seq_val.__len__() ''' trainInstances = vid_seq_train.__len__() train_params = [] if stage == 1: model = attentionModel(num_classes=num_classes, mem_size=memSize, CAM=CAM, device=device) model.train(False) for params in model.parameters(): params.requires_grad = False else: model = attentionModel(num_classes=num_classes, mem_size=memSize, CAM=CAM, device=device) model.load_state_dict(torch.load(stage1_dict)) model.train(False) for params in model.parameters(): params.requires_grad = False # for params in model.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.fc.parameters(): params.requires_grad = True train_params += [params] model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) for params in model.lstm_cell.parameters(): params.requires_grad = True train_params += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] model.lstm_cell.train(True) model.classifier.train(True) model.to(device) loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.Adam(train_params, lr=lr1, weight_decay=4e-5, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 min_accuracy = 0 for epoch in range(numEpochs): optim_scheduler.step() epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 model.lstm_cell.train(True) model.classifier.train(True) if stage == 2: model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) for i, (inputs, targets) in enumerate(train_loader): train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariable = inputs.permute(1, 0, 2, 3, 4).to(device) labelVariable = targets.to(device) trainSamples += inputs.size(0) output_label, _ = model(inputVariable) loss = loss_fn(output_label, labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.to(device)).sum() epoch_loss += loss.data.item() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = (numCorrTrain.data.item() / trainSamples) * 100 train_log_loss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format( epoch + 1, avg_loss, trainAccuracy)) #if val_data_dir is not None: if (epoch + 1) % 1 == 0: model.train(False) val_loss_epoch = 0 val_iter = 0 val_samples = 0 numCorr = 0 for j, (inputs, targets) in enumerate(val_loader): val_iter += 1 val_samples += inputs.size(0) inputVariable = inputs.permute(1, 0, 2, 3, 4).to(device) labelVariable = targets.to(device) output_label, _ = model(inputVariable) val_loss = loss_fn(output_label, labelVariable) val_loss_epoch += val_loss.data.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.to(device)).sum() val_accuracy = (numCorr.data.item() / val_samples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Valid: Epoch = {} | Loss {} | Accuracy = {}'.format( epoch + 1, avg_val_loss, val_accuracy)) val_log_loss.write('Val Loss after {} epochs = {}\n'.format( epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format( epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_rgb_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy '''else: if (epoch+1) % 10 == 0: save_path_model = (model_folder + '/model_rgb_state_dict_epoch' + str(epoch+1) + '.pth') torch.save(model.state_dict(), save_path_model) ''' train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close()
def main_run(version, stage, train_data_dir, stage1_dict, out_dir, seqLen, trainBatchSize, valBatchSize, numEpochs, lr1, decay_factor, decay_step, mem_size): num_classes = 61 model_folder = os.path.join("./", out_dir, version) if os.path.exists(model_folder): print('Directory {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # Train val partitioning train_usr = ["S1", "S3", "S4"] val_usr = ["S2"] # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose( [Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize]) vid_seq_train = makeDataset(train_data_dir, train_usr, spatial_transform=spatial_transform, seqLen=seqLen) train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=4, pin_memory=True) vid_seq_val = makeDataset(train_data_dir, val_usr, spatial_transform=Compose([Scale(256), CenterCrop(224), ToTensor(), normalize]), seqLen=seqLen) val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) train_params = [] # stage 1: train only lstm if stage == 1: model = attentionModel(num_classes=num_classes, mem_size=mem_size) model.train(False) for params in model.parameters(): params.requires_grad = False # stage 2: train lstm, layer4, spatial attention and final fc else: model = attentionModel(num_classes=num_classes, mem_size=mem_size) model.load_state_dict(torch.load(stage1_dict)) # pretrained model.train(False) for params in model.parameters(): params.requires_grad = False # for params in model.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.fc.parameters(): # fully connected layer params.requires_grad = True train_params += [params] model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) for params in model.lstm_cell.parameters(): # for both stages we train the lstm params.requires_grad = True train_params += [params] for params in model.classifier.parameters(): # for both stages we train the last classifier (after the lstm and avg pooling) params.requires_grad = True train_params += [params] model.lstm_cell.train(True) model.classifier.train(True) model.cuda() loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.Adam(train_params, lr=lr1, weight_decay=4e-5, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 min_accuracy = 0 for epoch in range(numEpochs): optim_scheduler.step() epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 model.lstm_cell.train(True) model.classifier.train(True) if stage == 2: model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) for i, (inputs, inputsF, targets) in enumerate(train_loader): train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariable = Variable(inputs.permute(1, 0, 2, 3, 4).to(DEVICE)) labelVariable = Variable(targets.to(DEVICE)) trainSamples += inputs.size(0) output_label, _ = model(inputVariable) loss = loss_fn(output_label, labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.to(DEVICE)).sum() # evaluating number of correct classifications epoch_loss += loss.data.item() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = (numCorrTrain.data.item() / trainSamples) train_log_loss.write('Training loss after {} epoch = {}\n'.format(epoch + 1, avg_loss)) # log file train_log_acc.write('Training accuracy after {} epoch = {}\n'.format(epoch + 1, trainAccuracy)) # log file print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format(epoch + 1, avg_loss, trainAccuracy)) if (epoch + 1) % VAL_FREQUENCY == 0: model.train(False) val_loss_epoch = 0 val_iter = 0 val_samples = 0 numCorr = 0 for j, (inputs, inputsF, targets) in enumerate(val_loader): val_iter += 1 val_samples += inputs.size(0) inputVariable = Variable(inputs.permute(1, 0, 2, 3, 4).to(DEVICE)) labelVariable = Variable(targets.to(DEVICE)) output_label, _ = model(inputVariable) val_loss = loss_fn(output_label, labelVariable) val_loss_epoch += val_loss.data.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.to(DEVICE)).sum() # evaluating number of correct classifications val_accuracy = (numCorr.data.item() / val_samples) avg_val_loss = val_loss_epoch / val_iter print('Val: Epoch = {} | Loss {} | Accuracy = {}'.format(epoch + 1, avg_val_loss, val_accuracy)) val_log_loss.write('Val Loss after {} epochs = {}\n'.format(epoch + 1, avg_val_loss)) # log file val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format(epoch + 1, val_accuracy)) # log file if val_accuracy > min_accuracy: save_path_model = ( model_folder + '/model_rgb_state_dict.pth') # every epoch, check if the val accuracy is improved, if so, save that model torch.save(model.state_dict(), save_path_model) # in that way, even if the model overfit, you will get always the best model min_accuracy = val_accuracy # in this way you don't have to care too much about the number of epochs train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close()
def main_run(version, flowModel, rgbModel, stackSize, seqLen, memSize, trainDatasetDir, outDir, trainBatchSize, valBatchSize, lr1, numEpochs, decay_step, decay_factor): num_classes = 61 # gtea61 dataset model_folder = os.path.join("./", outDir, version) # Create the dir print(f"Checking directory {model_folder}") if os.path.exists(model_folder): print('Dir {} exists!'.format(model_folder)) sys.exit() print(f"Creating directory{model_folder}") os.makedirs(model_folder) # Log files print(f"Creating log files") train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # ImageNet mean and std mean = [0.485, 0.456, 0.406] std = [0.229, 0.224, 0.225] # Train val partitioning train_usr = ["S1", "S3", "S4"] val_usr = ["S2"] normalize = Normalize(mean=mean, std=std) spatial_transform = Compose([Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize]) # train dataset print(f"Defining train dataset") vid_seq_train = makeDataset(trainDatasetDir, train_usr, spatial_transform, stackSize=stackSize, seqLen=seqLen) train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=4, pin_memory=True) # val dataset print(f"Defining validation dataset") vid_seq_val = makeDataset(trainDatasetDir, val_usr, spatial_transform=Compose([Scale(256), CenterCrop(224), ToTensor(), normalize]), stackSize=stackSize, phase="val", seqLen=seqLen) val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) valSamples = vid_seq_val.__len__() # model print("Building model") model = twoStreamAttentionModel(flowModel=flowModel, frameModel=rgbModel, stackSize=stackSize, memSize=memSize, # see twoStreamModel.py num_classes=num_classes) print("Setting trainable parameters") for params in model.parameters(): # initially freeze all layers params.requires_grad = False model.train(False) train_params = [] for params in model.classifier.parameters(): # unfreeze classifier layer (the layer that joins the two models outputs) params.requires_grad = True train_params += [params] for params in model.frameModel.lstm_cell.parameters(): # unfreeze lstm layer of the frame model train_params += [params] params.requires_grad = True for params in model.frameModel.resNet.layer4[0].conv1.parameters(): #unfreeze layer 4 params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.frameModel.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.frameModel.resNet.fc.parameters(): # unfreeze last fully connected layer of frame model params.requires_grad = True # (I still don't know why, because in the joining of the two models, this layer is skipped) train_params += [params] base_params = [] for params in model.flowModel.layer4.parameters(): # unfreeze layer 4 of flow model base_params += [params] params.requires_grad = True print("Moving model to GPU") model.to(DEVICE) trainSamples = vid_seq_train.__len__() min_accuracy = 0 print("Defining loss function, optimizer and scheduler") loss_fn = nn.CrossEntropyLoss() # loss function optimizer_fn = torch.optim.SGD([ # optimizer {'params': train_params}, {'params': base_params, 'lr': 1e-4}, # 1e-4 ], lr=lr1, momentum=0.9, weight_decay=5e-4) #scheduler optim_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_fn, step_size=decay_step, gamma=decay_factor) train_iter = 0 print("Training begun") # TRAIN PROCEDURE for epoch in range(numEpochs): optim_scheduler.step() epoch_loss = 0 numCorrTrain = 0 iterPerEpoch = 0 model.classifier.train(True) model.flowModel.layer4.train(True) start = time.time() for j, (inputFrame, inputMMaps, inputFlow, targets) in enumerate(train_loader): print(f"step {j} / {int(np.floor(trainSamples/trainBatchSize))}") train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() # put gradients to zero inputVariableFlow = Variable(inputFlow.to(DEVICE)) inputVariableFrame = Variable(inputFrame.permute(1, 0, 2, 3, 4).to(DEVICE)) labelVariable = Variable(targets.to(DEVICE)) #print("predict") output_label = model(inputVariableFlow, inputVariableFrame) # predict loss = loss_fn(F.log_softmax(output_label, dim=1), labelVariable) # compute loss #print("backprop") loss.backward() optimizer_fn.step() #print("accuracy") _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.to(DEVICE)).sum() # counting number of correct predictions epoch_loss += loss.data.item() avg_loss = epoch_loss / iterPerEpoch # computing average per epoch loss trainAccuracy = (numCorrTrain.item() / trainSamples) * 100 print('Average training loss after {} epoch = {} '.format(epoch + 1, avg_loss)) print('Training accuracy after {} epoch = {}% '.format(epoch + 1, trainAccuracy)) train_log_loss.write('Training loss after {} epoch = {}\n'.format(epoch + 1, avg_loss)) # log file train_log_acc.write('Training accuracy after {} epoch = {}\n'.format(epoch + 1, trainAccuracy)) # log file print(f"Elapsed : {time.time()-start}") # VALIDATION if (epoch + 1) % 5 == 0: model.train(False) val_loss_epoch = 0 val_iter = 0 numCorr = 0 for j, (inputFrame, inputMMaps, inputFlow, targets) in enumerate(val_loader): if j % 1 == 0: print(f"step {j} / {int(np.floor(vid_seq_val.__len__()/valBatchSize))}") val_iter += 1 inputVariableFlow = Variable(inputFlow.to(DEVICE)) inputVariableFrame = Variable(inputFrame.permute(1, 0, 2, 3, 4).to(DEVICE)) labelVariable = Variable(targets.to(DEVICE)) output_label = model(inputVariableFlow, inputVariableFrame) loss = loss_fn(F.log_softmax(output_label, dim=1), labelVariable) val_loss_epoch += loss.data.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == labelVariable.data).sum() val_accuracy = (numCorr.item() / valSamples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Val Loss after {} epochs, loss = {}'.format(epoch + 1, avg_val_loss)) print('Val Accuracy after {} epochs = {}%'.format(epoch + 1, val_accuracy)) val_log_loss.write('Val Loss after {} epochs = {}\n'.format(epoch + 1, avg_val_loss)) # log file val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format(epoch + 1, val_accuracy)) # log file if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_twoStream_state_dict.pth') # every epoch, check if the val accuracy is improved, if so, save that model torch.save(model.state_dict(), save_path_model) # in that way, even if the model overfit, you will get always the best model min_accuracy = val_accuracy # in this way you don't have to care too much about the number of epochs train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close()
def main(): global args global best_prec1 args = parser.parse_args() print('Training arguments:') for k, v in vars(args).items(): print('\t{}: {}'.format(k, v)) if args.data_name == 'ucf101': num_class = 101 elif args.data_name == 'hmdb51': num_class = 51 elif args.data_name == 'mine': num_class = 2 else: raise ValueError('Unknown dataset ' + args.data_name) model = Model(num_class, args.num_segments, args.representation, base_model=args.arch) print(model) if 'resnet3D' in args.arch: train_crop_min_ratio = 0.75 train_crop_min_scale = 0.25 mean = [0.4345, 0.4051, 0.3775] std = [0.2768, 0.2713, 0.2737] value_scale = 1 train_transform = Compose([ RandomResizedCrop( model.crop_size, (train_crop_min_scale, 1.0), (train_crop_min_ratio, 1.0 / train_crop_min_ratio)), RandomHorizontalFlip(), ToTensor(), ScaleValue(value_scale), Normalize(mean, std) ]) test_trainsform = Compose([ Resize(model.crop_size), CenterCrop(model.crop_size), ToTensor(), # range [0, 255] -> [0.0,1.0] ScaleValue(1), Normalize(mean, std) ]) train_loader = torch.utils.data.DataLoader( CoviarDataSet( args.data_root, args.data_name, video_list=args.train_list, num_segments=args.num_segments, representation=args.representation, transform=model.get_augmentation(), #train_transform, is_train=True, accumulate=(not args.no_accumulation), model_name=args.arch), batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True, worker_init_fn=worker_init_fn) val_loader = torch.utils.data.DataLoader( CoviarDataSet( args.data_root, args.data_name, video_list=args.test_list, num_segments=args.num_segments, representation=args.representation, transform=torchvision.transforms.Compose([ GroupScale(int(model.scale_size)), GroupCenterCrop(model.crop_size) ]), #test_trainsform, is_train=True, accumulate=(not args.no_accumulation), model_name=args.arch), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True, worker_init_fn=worker_init_fn) model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda() cudnn.benchmark = True params_dict = dict(model.named_parameters()) params = [] for key, value in params_dict.items(): decay_mult = 0.0 if 'bias' in key else 1.0 if ('module.base_model.conv1' in key or 'module.base_model.bn1' in key or 'data_bn' in key) and args.representation in ['mv', 'residual']: lr_mult = 0.1 elif '.fc.' in key: lr_mult = 1.0 else: lr_mult = 0.01 params += [{ 'params': value, 'lr': args.lr, 'lr_mult': lr_mult, 'decay_mult': decay_mult }] #optimizer = torch.optim.SGD(params, weight_decay=0.001, momentum=0.9, nesterov=False) #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=10) optimizer = torch.optim.Adam(params, weight_decay=args.weight_decay, eps=0.001) criterion = torch.nn.CrossEntropyLoss().cuda() for epoch in range(args.epochs): cur_lr = adjust_learning_rate(optimizer, epoch, args.lr_steps, args.lr_decay) #cur_lr = get_lr(optimizer) train(train_loader, model, criterion, optimizer, epoch, cur_lr) #prec1, prev_val_loss = validate(val_loader, model, criterion) #scheduler.step(prev_val_loss) if epoch % args.eval_freq == 0 or epoch == args.epochs - 1: prec1, _ = validate(val_loader, model, criterion) # 紀錄訓練歷程 np.savez("train_history/train_history.npz", loss=np.array(train_loss), top1=np.array(train_prec), lr=np.array(train_lr)) np.savez("train_history/valid_history.npz", loss=np.array(valid_loss), top1=np.array(valid_prec)) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) if is_best or epoch % SAVE_FREQ == 0: save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, }, is_best, filename='checkpoint.pth.tar')
def main_run(dataset, stage, root_dir, out_dir, seqLen, trainBatchSize, numEpochs, lr1, decay_factor, decay_step, memSize, outPool_size, split, evalInterval): mean = [0.485, 0.456, 0.406] std = [0.229, 0.224, 0.225] normalize = Normalize(mean=mean, std=std) stage = stage test_split = split seqLen = seqLen memSize = memSize c_cam_classes = outPool_size dataset = dataset best_acc = 0 if stage == 1: trainBatchSize = trainBatchSize testBatchSize = trainBatchSize lr1 = lr1 decay_factor = decay_factor decay_step = decay_step numEpochs = numEpochs elif stage == 2: trainBatchSize = trainBatchSize testBatchSize = trainBatchSize lr1 = lr1 decay_factor = decay_factor decay_step = decay_step numEpochs = numEpochs if dataset == 'gtea_61': num_classes = 61 elif dataset == 'gtea_71': num_classes = 71 elif dataset == 'egtea_gaze+': num_classes = 106 else: print('Wrong dataset') sys.exit() dataset_dir = os.path.join(root_dir, dataset) model_folder = os.path.join('.', out_dir, dataset, str(test_split)) if not os.path.exists(model_folder): os.makedirs(model_folder) note_fl = open(model_folder + '/note.txt', 'w') note_fl.write('Number of Epochs = {}\n' 'lr = {}\n' 'Train Batch Size = {}\n' 'Sequence Length = {}\n' 'Decay steps = {}\n' 'Decay factor = {}\n' 'Memory size = {}\n' 'Memory cam classes = {}\n'.format(numEpochs, lr1, trainBatchSize, seqLen, decay_step, decay_factor, memSize, c_cam_classes)) note_fl.close() # Log files writer = SummaryWriter(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') train_log_loss_batch = open((model_folder + '/train_log_loss_batch.txt'), 'w') test_log_loss = open((model_folder + '/test_log_loss.txt'), 'w') test_log_acc = open((model_folder + '/test_log_acc.txt'), 'w') spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) print('Preparing dataset...') if dataset == 'egtea_gaze+': trainDatasetF, testDatasetF, trainLabels, testLabels, trainNumFrames, testNumFrames = gen_split_egtea_gazePlus( dataset_dir, test_split) else: trainDatasetF, testDatasetF, trainLabels, testLabels, trainNumFrames, testNumFrames, _ = gen_split( dataset_dir, test_split) vid_seq_train = makeDataset(trainDatasetF, trainLabels, trainNumFrames, spatial_transform=spatial_transform, fmt='.jpg', seqLen=seqLen) print('Number of train samples = {}'.format(vid_seq_train.__len__())) train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, num_workers=4, pin_memory=True) vid_seq_test = makeDataset(testDatasetF, testLabels, testNumFrames, spatial_transform=Compose([ Scale(256), CenterCrop(224), ToTensor(), normalize ]), fmt='.jpg', seqLen=seqLen) print('Number of test samples = {}'.format(vid_seq_test.__len__())) test_loader = torch.utils.data.DataLoader(vid_seq_test, batch_size=testBatchSize, shuffle=False, num_workers=2, pin_memory=True) train_params = [] if stage == 1: model = attentionModel(num_classes=num_classes, mem_size=memSize, c_cam_classes=c_cam_classes) model.train(False) for params in model.parameters(): params.requires_grad = False elif stage == 2: model = attentionModel(num_classes=num_classes, mem_size=memSize, c_cam_classes=c_cam_classes) checkpoint_path = os.path.join( model_folder, 'last_checkpoint_stage' + str(1) + '.pth.tar') if os.path.exists(checkpoint_path): print('Loading weights from checkpoint file {}'.format( checkpoint_path)) else: print('Checkpoint file {} does not exist'.format(checkpoint_path)) sys.exit() last_checkpoint = torch.load(checkpoint_path) model.load_state_dict(last_checkpoint['model_state_dict']) model.train(False) for params in model.parameters(): params.requires_grad = False for params in model.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.fc.parameters(): params.requires_grad = True train_params += [params] for params in model.lsta_cell.parameters(): params.requires_grad = True train_params += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] model.classifier.train(True) model.cuda() loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.Adam(train_params, lr=lr1, weight_decay=5e-4, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 for epoch in range(numEpochs): optim_scheduler.step() epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 model.classifier.train(True) writer.add_scalar('lr', optimizer_fn.param_groups[0]['lr'], epoch + 1) for i, (inputs, targets) in enumerate(train_loader): train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariable = Variable(inputs.permute(1, 0, 2, 3, 4).cuda()) labelVariable = Variable(targets.cuda()) trainSamples += inputs.size(0) output_label, _ = model(inputVariable) loss = loss_fn(output_label, labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.cuda()).sum() if train_iter % 10 == 0: print('Training loss after {} iterations = {} '.format( train_iter, loss.data[0])) train_log_loss_batch.write( 'Training loss after {} iterations = {}\n'.format( train_iter, loss.data[0])) writer.add_scalar('train/iter_loss', loss.data[0], train_iter) epoch_loss += loss.data[0] avg_loss = epoch_loss / iterPerEpoch trainAccuracy = (numCorrTrain / trainSamples) * 100 print('Average training loss after {} epoch = {} '.format( epoch + 1, avg_loss)) print('Training accuracy after {} epoch = {}% '.format( epoch + 1, trainAccuracy)) writer.add_scalar('train/epoch_loss', avg_loss, epoch + 1) writer.add_scalar('train/accuracy', trainAccuracy, epoch + 1) train_log_loss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) save_path_model = os.path.join( model_folder, 'last_checkpoint_stage' + str(stage) + '.pth.tar') save_file = { 'epoch': epoch + 1, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer_fn.state_dict(), 'best_acc': best_acc, } torch.save(save_file, save_path_model) if (epoch + 1) % evalInterval == 0: print('Testing...') model.train(False) test_loss_epoch = 0 test_iter = 0 test_samples = 0 numCorr = 0 for j, (inputs, targets) in enumerate(test_loader): print('testing inst = {}'.format(j)) test_iter += 1 test_samples += inputs.size(0) inputVariable = Variable(inputs.permute(1, 0, 2, 3, 4).cuda(), volatile=True) labelVariable = Variable(targets.cuda(async=True), volatile=True) output_label, _ = model(inputVariable) test_loss = loss_fn(output_label, labelVariable) test_loss_epoch += test_loss.data[0] _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.cuda()).sum() test_accuracy = (numCorr / test_samples) * 100 avg_test_loss = test_loss_epoch / test_iter print('Test Loss after {} epochs, loss = {}'.format( epoch + 1, avg_test_loss)) print('Test Accuracy after {} epochs = {}%'.format( epoch + 1, test_accuracy)) writer.add_scalar('test/epoch_loss', avg_test_loss, epoch + 1) writer.add_scalar('test/accuracy', test_accuracy, epoch + 1) test_log_loss.write('Test Loss after {} epochs = {}\n'.format( epoch + 1, avg_test_loss)) test_log_acc.write('Test Accuracy after {} epochs = {}%\n'.format( epoch + 1, test_accuracy)) if test_accuracy > best_acc: best_acc = test_accuracy save_path_model = os.path.join( model_folder, 'best_checkpoint_stage' + str(stage) + '.pth.tar') save_file = { 'epoch': epoch + 1, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer_fn.state_dict(), 'best_acc': best_acc, } torch.save(save_file, save_path_model) train_log_loss.close() train_log_acc.close() test_log_acc.close() train_log_loss_batch.close() test_log_loss.close() writer.export_scalars_to_json(model_folder + "/all_scalars.json") writer.close()
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size, crop_positions=['c']) elif opt.train_crop == 'driver focus': crop_method = DriverFocusCrop(opt.scales, opt.sample_size) train_spatial_transform = Compose([ Scale(opt.sample_size), ToTensor(opt.norm_value) #, norm_method ]) train_temporal_transform = UniformIntervalCrop(opt.sample_duration, opt.interval) train_target_transform = Compose([ Scale(opt.sample_size), ToTensor(opt.norm_value) #, norm_method ]) train_horizontal_flip = RandomHorizontalFlip() training_data = get_training_set(opt, train_spatial_transform, train_horizontal_flip, train_temporal_transform, train_target_transform) train_loader = torch.utils.data.DataLoader(training_data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.n_threads, pin_memory=True) train_logger = Logger(os.path.join(opt.result_path, 'convlstm-train.log'), ['epoch', 'loss', 'lr']) train_batch_logger = Logger( os.path.join(opt.result_path, 'convlstm-train_batch.log'), ['epoch', 'batch', 'iter', 'loss', 'lr'])
def main_run(dataset, flowModel, rgbModel, stage, seqLen, memSize, trainDatasetDir, valDatasetDir, outDir, trainBatchSize, valBatchSize, lr1, numEpochs, decay_step, decay_factor): if dataset == 'gtea61': num_classes = 61 elif dataset == 'gtea71': num_classes = 71 elif dataset == 'gtea_gaze': num_classes = 44 elif dataset == 'egtea': num_classes = 106 else: print('Dataset not found') sys.exit() model_folder = os.path.join('./', outDir, dataset, 'NewtwoStream',str(stage)) # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Dir {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files writer = SummaryWriter(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') mean = [0.485, 0.456, 0.406] std = [0.229, 0.224, 0.225] normalize = Normalize(mean=mean, std=std) spatial_transform = Compose([Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize]) vid_seq_train = makeDataset(trainDatasetDir,spatial_transform=spatial_transform, sequence=False, numSeg=1, fmt='.png', seqLen=seqLen, frame_div=True) train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=4, pin_memory=True) if valDatasetDir is not None: vid_seq_val = makeDataset(valDatasetDir, spatial_transform=Compose([Scale(256), CenterCrop(224), ToTensor(), normalize]), sequence=False, numSeg=1, fmt='.png', phase='Test', seqLen=seqLen, frame_div=True) val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) valSamples = vid_seq_val.__len__() train_params = [] if stage == 1: model = attentionModel_flow(num_classes=num_classes,frameModel=rgbModel, mem_size=memSize) model.train(False) for params in model.parameters(): params.requires_grad = False else: model = attentionModel_flow(num_classes=num_classes, mem_size=memSize) model.load_state_dict(torch.load(flowModel)) model.train(False) for params in model.parameters(): params.requires_grad = False # for params in model.flowResNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.flowResNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.flowResNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.flowResNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.flowResNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.flowResNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.flowResNet.fc_action.parameters(): params.requires_grad = True train_params += [params] model.flowResNet.layer4[0].conv1.train(True) model.flowResNet.layer4[0].conv2.train(True) model.flowResNet.layer4[1].conv1.train(True) model.flowResNet.layer4[1].conv2.train(True) model.flowResNet.layer4[2].conv1.train(True) model.flowResNet.layer4[2].conv2.train(True) model.flowResNet.fc_action.train(True) for params in model.lstm_cell.parameters(): params.requires_grad = True train_params += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] model.lstm_cell.train(True) model.classifier.train(True) model.cuda() loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.Adam(train_params, lr=lr1, weight_decay=4e-5, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones=decay_step, gamma=decay_factor) trainSamples = vid_seq_train.__len__() min_accuracy = 0 train_iter = 0 for epoch in range(numEpochs): model.lstm_cell.train(True) model.classifier.train(True) if stage == 2: model.flowResNet.layer4[0].conv1.train(True) model.flowResNet.layer4[0].conv2.train(True) model.flowResNet.layer4[1].conv1.train(True) model.flowResNet.layer4[1].conv2.train(True) model.flowResNet.layer4[2].conv1.train(True) model.flowResNet.layer4[2].conv2.train(True) model.flowResNet.fc_action.train(True) epoch_loss = 0 numCorrTrain = 0 iterPerEpoch = 0 for j, (inputFlow, inputFrame, targets) in enumerate(train_loader): train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariableFlow = inputFlow.permute(1, 0, 2, 3, 4).cuda() inputVariableFrame = inputFrame.permute(1, 0, 2, 3, 4).cuda() labelVariable = targets.cuda() output_label,_ = model(inputVariableFlow, inputVariableFrame) loss = loss_fn(output_label, labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += torch.sum(predicted == labelVariable.data).data.item() epoch_loss += loss.item() optim_scheduler.step() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = (numCorrTrain / trainSamples) * 100 print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format(epoch+1, avg_loss, trainAccuracy)) writer.add_scalar('train/epoch_loss', avg_loss, epoch + 1) writer.add_scalar('train/accuracy', trainAccuracy, epoch + 1) train_log_loss.write('Training loss after {} epoch = {}\n'.format(epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format(epoch + 1, trainAccuracy)) if valDatasetDir is not None: if (epoch + 1) % 1 == 0: model.train(False) val_loss_epoch = 0 val_iter = 0 numCorr = 0 for j, (inputFlow, inputFrame, targets) in enumerate(val_loader): val_iter += 1 inputVariableFlow = inputFlow.permute(1, 0, 2, 3, 4).cuda() inputVariableFrame = inputFrame.permute(1, 0, 2, 3, 4).cuda() labelVariable = targets.cuda() output_label,_ = model(inputVariableFlow, inputVariableFrame) loss = loss_fn(output_label, labelVariable) val_loss_epoch += loss.item() _, predicted = torch.max(output_label.data, 1) numCorr += torch.sum(predicted == labelVariable.data).data.item() val_accuracy = (numCorr / valSamples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Val: Epoch = {} | Loss {} | Accuracy = {}'.format(epoch + 1, avg_val_loss, val_accuracy)) writer.add_scalar('val/epoch_loss', avg_val_loss, epoch + 1) writer.add_scalar('val/accuracy', val_accuracy, epoch + 1) val_log_loss.write('Val Loss after {} epochs = {}\n'.format(epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format(epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_twoStream_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy else: if (epoch + 1) % 10 == 0: save_path_model = (model_folder + '/model_twoStream_state_dict_epoch' + str(epoch + 1) + '.pth') torch.save(model.state_dict(), save_path_model) train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close() writer.export_scalars_to_json(model_folder + "/all_scalars.json") writer.close()
def main_run(dataset, stage, train_data_dir, val_data_dir, stage1_dict, out_dir, seqLen, trainBatchSize, valBatchSize, numEpochs, lr1, decay_factor, decay_step, memSize, regression, rloss, debug, verbose, CAM): # GTEA 61 num_classes = 61 # Train/Validation/Test split train_splits = ["S1", "S3", "S4"] val_splits = ["S2"] if debug: n_workers = 0 device = 'cpu' else: n_workers = 4 device = 'cuda' model_folder = os.path.join( './', out_dir, dataset, 'rgb', 'stage' + str(stage)) # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Directory {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), # ToTensor(), # normalize ]) transform_rgb = Compose([ToTensor(), normalize]) transform_MS = Compose([Resize((7, 7)), ToTensor()]) vid_seq_train = makeDataset(train_data_dir, splits=train_splits, spatial_transform=spatial_transform, transform_rgb=transform_rgb, transform_MS=transform_MS, seqLen=seqLen, fmt='.png', regression=regression) train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=n_workers, pin_memory=True) vid_seq_val = makeDataset(train_data_dir, splits=val_splits, spatial_transform=Compose( [Scale(256), CenterCrop(224)]), transform_rgb=transform_rgb, transform_MS=transform_MS, seqLen=seqLen, fmt='.png', regression=regression, verbose=False) val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=n_workers, pin_memory=True) valInstances = vid_seq_val.__len__() ''' if val_data_dir is not None: vid_seq_val = makeDataset(val_data_dir, spatial_transform=Compose([Scale(256), CenterCrop(224), ToTensor(), normalize]), seqLen=seqLen, fmt='.jpg') val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) valInstances = vid_seq_val.__len__() ''' trainInstances = vid_seq_train.__len__() train_params = [] if stage == 1: if regression: model = SelfSupervisedAttentionModel(num_classes=num_classes, mem_size=memSize, n_channels=1) else: model = SelfSupervisedAttentionModel(num_classes=num_classes, mem_size=memSize) model.train(False) for params in model.parameters(): params.requires_grad = False else: if regression: model = SelfSupervisedAttentionModel(num_classes=num_classes, mem_size=memSize, n_channels=1) else: model = SelfSupervisedAttentionModel(num_classes=num_classes, mem_size=memSize) model.load_state_dict(torch.load(stage1_dict), strict=False) model.train(False) for params in model.parameters(): params.requires_grad = False # for params in model.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.fc.parameters(): params.requires_grad = True train_params += [params] model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) # Add params from ms_module for params in model.ms_module.parameters(): params.requires_grad = True train_params += [params] for params in model.lstm_cell.parameters(): params.requires_grad = True train_params += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] model.lstm_cell.train(True) model.classifier.train(True) model.ms_module.train(True) model.to(device) # wandb.init(project="first_person_action_recognition") loss_fn = nn.CrossEntropyLoss() if regression: if rloss == 'MSE': # Mean Squared Error loss loss_ms_fn = nn.MSELoss() # it should work elif rloss == 'L1': # L1 loss loss_ms_fn = nn.L1Loss() elif rloss == 'SmoothL1': # Huber Loss or Smooth L1 Loss loss_ms_fn = nn.SmoothL1Loss() elif rloss == 'KLdiv': # Kullback-Leiber Loss loss_ms_fn = nn.KLDivLoss() else: # classification loss_ms_fn = nn.CrossEntropyLoss() # TODO: check paper Planamente optimizer_fn = torch.optim.Adam(train_params, lr=lr1, weight_decay=4e-5, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 min_accuracy = 0 for epoch in range(numEpochs): epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 #model.train(True) model.lstm_cell.train(True) model.classifier.train(True) if stage == 2: model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) model.ms_module.train(True) for i, (inputsRGB, inputsMS, targets) in enumerate(train_loader): # Inputs: # - inputsRGB : the rgb frame input # Labels : # - inputsMS : the motion task label # - targets : output train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariable = inputsRGB.permute(1, 0, 2, 3, 4).to(device) labelVariable = targets.to(device) msVariable = inputsMS.to(device) trainSamples += inputsRGB.size(0) output_label, _, output_ms = model(inputVariable, device) loss_c = loss_fn(output_label, labelVariable) if regression: msVariable = torch.reshape( msVariable, (seqLen * 7 * 7, msVariable.size(0))) output_ms = torch.sigmoid(output_ms) output_ms = torch.reshape(output_ms, (seqLen * 7 * 7, output_ms.size(0))) else: # classification task msVariable = torch.reshape( msVariable, (seqLen * 7 * 7, msVariable.size(0))).long() output_ms = torch.reshape( output_ms, (seqLen * 7 * 7, 2, output_ms.size(0))) # loss_ms = loss_ms_fn(output_ms, msVariable) loss = loss_c + loss_ms if verbose: print(loss_c) print(loss_ms) print(loss) print() # loss = loss_fn(output_label, labelVariable) + loss_ms_fn(output_ms, inputsMS) # TODO (forse): invertire 0 e 1 dim per inputsMS # output1 = F.softmax(torch.reshape(output_ms, (32, 7, 2, 7*7))[0, 0, :, :], dim=0) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.to(device)).sum() epoch_loss += loss.data.item() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = (numCorrTrain.data.item() / trainSamples) * 100 train_log_loss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format( epoch + 1, avg_loss, trainAccuracy)) # VALIDATION PHASE #if val_data_dir is not None: if (epoch + 1) % 1 == 0: model.train(False) val_loss_epoch = 0 val_iter = 0 val_samples = 0 numCorr = 0 for j, (inputsRGB, inputsMS, targets) in enumerate(val_loader): val_iter += 1 val_samples += inputsRGB.size(0) inputVariable = inputsRGB.permute(1, 0, 2, 3, 4).to( device) # la permutazione è a solo scopo di computazione labelVariable = targets.to(device) msVariable = inputsMS.to(device) output_label, _, output_ms = model(inputVariable, device) loss_c = loss_fn(output_label, labelVariable) if regression: msVariable = torch.reshape( msVariable, (seqLen * 7 * 7, msVariable.size(0))) output_ms = torch.sigmoid(output_ms) output_ms = torch.reshape( output_ms, (seqLen * 7 * 7, output_ms.size(0))) else: # classification task msVariable = torch.reshape( msVariable, (seqLen * 7 * 7, msVariable.size(0))).long() output_ms = torch.reshape( output_ms, (seqLen * 7 * 7, 2, output_ms.size(0))) loss_ms = loss_ms_fn(output_ms, msVariable) val_loss = loss_c + loss_ms # val_loss = loss_fn(output_label, labelVariable) # TODO: add ms Loss val_loss_epoch += val_loss.data.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.to(device)).sum() val_accuracy = (numCorr.data.item() / val_samples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Valid: Epoch = {} | Loss {} | Accuracy = {}'.format( epoch + 1, avg_val_loss, val_accuracy)) val_log_loss.write('Val Loss after {} epochs = {}\n'.format( epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format( epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_rgb_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy '''else: if (epoch+1) % 10 == 0: save_path_model = (model_folder + '/model_rgb_state_dict_epoch' + str(epoch+1) + '.pth') torch.save(model.state_dict(), save_path_model) ''' optim_scheduler.step() train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close()
def get_train_utils(opt, model_parameters): assert opt.train_crop in ['random', 'corner', 'center'] spatial_transform = [] if opt.train_crop == 'random': spatial_transform.append( RandomResizedCrop( opt.sample_size, (opt.train_crop_min_scale, 1.0), (opt.train_crop_min_ratio, 1.0 / opt.train_crop_min_ratio))) elif opt.train_crop == 'corner': scales = [1.0] scale_step = 1 / (2**(1 / 4)) for _ in range(1, 5): scales.append(scales[-1] * scale_step) spatial_transform.append(MultiScaleCornerCrop(opt.sample_size, scales)) elif opt.train_crop == 'center': spatial_transform.append(Resize(opt.sample_size)) spatial_transform.append(CenterCrop(opt.sample_size)) normalize = get_normalize_method(opt.mean, opt.std, opt.no_mean_norm, opt.no_std_norm) if not opt.no_hflip: spatial_transform.append(RandomHorizontalFlip()) spatial_transform.append(ToArray()) if opt.colorjitter: spatial_transform.append(ColorJitter()) if opt.input_type == 'flow': spatial_transform.append(PickFirstChannels(n=2)) spatial_transform.append(ScaleValue(opt.value_scale)) spatial_transform.append(normalize) spatial_transform = Compose(spatial_transform) assert opt.train_t_crop in ['random', 'center'] temporal_transform = [] if opt.sample_t_stride > 1: temporal_transform.append(TemporalSubsampling(opt.sample_t_stride)) if opt.train_t_crop == 'random': temporal_transform.append(TemporalRandomCrop(opt.sample_duration)) elif opt.train_t_crop == 'center': temporal_transform.append(TemporalCenterCrop(opt.sample_duration)) temporal_transform = TemporalCompose(temporal_transform) train_data = get_training_data(opt.video_path, opt.annotation_path, opt.dataset, opt.input_type, opt.file_type, spatial_transform, temporal_transform) train_loader = paddle.batch(train_data.reader, batch_size=opt.batch_size) train_logger = Logger(opt.result_path / 'train.log', ['epoch', 'loss', 'acc', 'lr']) train_batch_logger = Logger( opt.result_path / 'train_batch.log', ['epoch', 'batch', 'iter', 'loss', 'acc', 'lr']) assert opt.lr_scheduler in ['plateau', 'multistep'] assert not (opt.lr_scheduler == 'plateau' and opt.no_val) if opt.lr_scheduler == 'plateau': scheduler = ReduceLROnPlateau(learning_rate=opt.learning_rate, mode='min', patience=opt.plateau_patience) else: scheduler = MultiStepDecay(learning_rate=opt.learning_rate, milestones=opt.multistep_milestones) optimizer = fluid.optimizer.MomentumOptimizer( learning_rate=scheduler, momentum=opt.momentum, parameter_list=model_parameters, use_nesterov=opt.nesterov, regularization=fluid.regularizer.L2Decay( regularization_coeff=opt.weight_decay)) return (train_loader, train_logger, train_batch_logger, optimizer, scheduler)
def main_run(dataset, stage, train_data_dir, val_data_dir, stage1_dict, out_dir, seqLen, trainBatchSize, valBatchSize, numEpochs, lr1, decay_factor, decay_step, memSize, regressor): if dataset == 'gtea61': num_classes = 61 elif dataset == 'gtea71': num_classes = 71 elif dataset == 'gtea_gaze': num_classes = 44 elif dataset == 'egtea': num_classes = 106 else: print('Dataset not found') sys.exit() model_folder = os.path.join('./', out_dir, dataset, 'MS',str(stage)) # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Directory {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files writer = SummaryWriter(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') train_log_loss_ms= open((model_folder + '/train_log_loss_ms.txt'), 'w') val_log_loss_ms = open((model_folder + '/val_log_loss_ms.txt'), 'w') train_log_acc_ms= open((model_folder + '/train_log_acc_ms.txt'), 'w') val_log_acc_ms = open((model_folder + '/val_log_acc_ms.txt'), 'w') # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224)]) vid_seq_train = makeDataset(train_data_dir, spatial_transform=spatial_transform, seqLen=seqLen, fmt='.png',phase='train', regressor=regressor) train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=4, pin_memory=True) if val_data_dir is not None: vid_seq_val = makeDataset(val_data_dir, spatial_transform=Compose([Scale(256), CenterCrop(224)]), seqLen=seqLen, fmt='.png',phase='test', regressor=regressor) val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) valInstances = vid_seq_val.__len__() trainInstances = vid_seq_train.__len__() train_params = [] if stage == 1: model = attentionModel_ml(num_classes=num_classes, mem_size=memSize, regressor=regressor) model.train(False) for params in model.parameters(): params.requires_grad = False else: model = attentionModel_ml(num_classes=num_classes, mem_size=memSize, regressor=regressor) model.load_state_dict(torch.load(stage1_dict),strict=False) model.train(False) for params in model.parameters(): params.requires_grad = False # for params in model.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.resNet.fc.parameters(): params.requires_grad = True train_params += [params] for params in model.conv.parameters(): params.requires_grad = True train_params += [params] for params in model.clas.parameters(): params.requires_grad = True train_params += [params] model.conv.train(True) model.clas.train(True) model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) for params in model.lstm_cell.parameters(): params.requires_grad = True train_params += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] model.lstm_cell.train(True) model.classifier.train(True) model.cuda() loss_fn = nn.CrossEntropyLoss() loss_fms = nn.NLLLoss() loss_reg = nn.MSELoss() optimizer_fn = torch.optim.Adam(train_params, lr=lr1, weight_decay=4e-5, eps=1e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 min_accuracy = 0 for epoch in range(numEpochs): epoch_loss = 0 numCorrTrain = 0 numCorrTrain_ms = 0 trainSamples = 0 iterPerEpoch = 0 epoch_loss_ms = 0 model.lstm_cell.train(True) model.classifier.train(True) writer.add_scalar('lr', optimizer_fn.param_groups[0]['lr'], epoch+1) if stage == 2: model.conv.train(True) model.clas.train(True) model.resNet.layer4[0].conv1.train(True) model.resNet.layer4[0].conv2.train(True) model.resNet.layer4[1].conv1.train(True) model.resNet.layer4[1].conv2.train(True) model.resNet.layer4[2].conv1.train(True) model.resNet.layer4[2].conv2.train(True) model.resNet.fc.train(True) for i, (inputs ,binary_map, targets) in enumerate(train_loader): train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariable = Variable(inputs.permute(1, 0, 2, 3, 4).cuda()) labelVariable = Variable(targets.cuda()) trainSamples += inputs.size(0) output_label, output_ms = model(inputVariable) loss = loss_fn(output_label, labelVariable) if stage==2 : loss.backward(retain_graph=True) else: loss.backward() if regressor == 0: binary_map = Variable(binary_map.permute(1, 0, 2, 3, 4).type(torch.LongTensor).cuda()) output_ms = output_ms.view(-1,2) elif regressor == 1: binary_map = Variable(binary_map.permute(1, 0, 2, 3, 4).cuda()) output_ms = output_ms.view(-1) binary_map =binary_map.contiguous().view(-1) if stage==2: if regressor == 1: loss_ms=loss_reg(output_ms, binary_map) loss_ms.backward() epoch_loss_ms+=loss_ms.item() elif regressor == 0: loss_ms=loss_fn(output_ms, binary_map) loss_ms.backward() _, predicted = torch.max(output_ms.data, 1) numCorrTrain_ms += torch.sum(predicted == binary_map.data).data.item() epoch_loss_ms+=loss_ms.item() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += torch.sum(predicted == labelVariable.data).data.item() epoch_loss += loss.item() avg_loss = epoch_loss/iterPerEpoch if stage ==2: trainAccuracy = (numCorrTrain_ms / trainSamples) * 100 avg_loss_ms= epoch_loss_ms/iterPerEpoch #avg_loss = avg_loss + avg_loss_ms train_log_loss_ms.write('Train Loss MS after {} epochs = {}\n'.format(epoch + 1, avg_loss_ms)) if regressor == 0:train_log_acc_ms.write('Train Accuracy after {} epochs = {}%\n'.format(epoch + 1, trainAccuracy)) trainAccuracy = (numCorrTrain / trainSamples) * 100 print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format(epoch+1, avg_loss, trainAccuracy)) writer.add_scalar('train/epoch_loss', avg_loss, epoch+1) writer.add_scalar('train/accuracy', trainAccuracy, epoch+1) train_log_loss.write('Train Loss after {} epochs = {}\n'.format(epoch + 1, avg_loss)) train_log_acc.write('Train Accuracy after {} epochs = {}%\n'.format(epoch + 1, trainAccuracy)) if val_data_dir is not None: if (epoch+1) % 1 == 0: model.train(False) val_loss_epoch = 0 val_iter = 0 val_samples = 0 numCorr = 0 numCorr_ms = 0 epoch_loss_ms_val=0 for j, (inputs, binary_map, targets) in enumerate(val_loader): val_iter += 1 val_samples += inputs.size(0) inputVariable = Variable(inputs.permute(1, 0, 2, 3, 4).cuda(), volatile=True) labelVariable = Variable(targets.cuda(async=True), volatile=True) output_label, output_ms = model(inputVariable) val_loss = loss_fn(output_label, labelVariable) val_loss_epoch += val_loss.item() if regressor == 0: binary_map = Variable(binary_map.permute(1, 0, 2, 3, 4).type(torch.LongTensor).cuda()) output_ms = output_ms.view(-1,2) elif regressor == 1: binary_map = Variable(binary_map.permute(1, 0, 2, 3, 4).cuda()) output_ms = output_ms.view(-1) binary_map =binary_map.contiguous().view(-1) if stage==2: if regressor == 1: loss_ms=loss_reg(output_ms, binary_map) epoch_loss_ms_val+=loss_ms.item() elif regressor == 0: loss_ms=loss_fn(output_ms, binary_map) _, predicted = torch.max(output_ms.data, 1) numCorr_ms += torch.sum(predicted == binary_map.data).data.item() epoch_loss_ms_val+=loss_ms.item() _, predicted = torch.max(output_label.data, 1) numCorr += torch.sum(predicted == labelVariable.data).data.item() avg_val_loss = val_loss_epoch / val_iter if stage ==2: avg_loss_ms= epoch_loss_ms_val/ val_iter val_accuracy = (numCorr_ms / val_samples) * 100 #avg_loss = avg_loss + avg_loss_ms val_log_loss_ms.write('Val Loss MS after {} epochs = {}\n'.format(epoch + 1, avg_loss_ms)) if regressor == 0:val_log_acc_ms.write('Val Accuracy after {} epochs = {}%\n'.format(epoch + 1, val_accuracy)) val_accuracy = (numCorr / val_samples) * 100 print('Val: Epoch = {} | Loss {} | Accuracy = {}'.format(epoch + 1, avg_val_loss, val_accuracy)) writer.add_scalar('val/epoch_loss', avg_val_loss, epoch + 1) writer.add_scalar('val/accuracy', val_accuracy, epoch + 1) val_log_loss.write('Val Loss after {} epochs = {}\n'.format(epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format(epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_ms_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy else: if (epoch+1) % 10 == 0: save_path_model = (model_folder + '/model_ms_state_dict_epoch' + str(epoch+1) + '.pth') torch.save(model.state_dict(), save_path_model) train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close() train_log_loss_ms.close() val_log_loss_ms.close() writer.export_scalars_to_json(model_folder + "/all_scalars.json") writer.close() optim_scheduler.step()
# transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), # ]) # transform_test = transforms.Compose([ # transforms.ToTensor(), # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), # ]) data_path = args.datapath seqLen=args.seqLen testBatchSize=1 trainX, trainY, testX, testY = make_split(data_path) mean=[0.485, 0.456, 0.406] std=[0.229, 0.224, 0.225] normalize = Normalize(mean=mean, std=std) spatial_transform = Compose([Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize]) vidSeqTrain = makeDataset(trainX, trainY, spatial_transform=spatial_transform, seqLen=seqLen) trainLoader = torch.utils.data.DataLoader(vidSeqTrain, batch_size=args.trainBatchSize, shuffle=True, num_workers=0) test_spatial_transform = Compose([Scale(256), CenterCrop(224), FlippedImagesTest(mean=mean, std=std)]) vidSeqTest = makeDataset(testX, testY, seqLen=seqLen, spatial_transform=test_spatial_transform) testLoader = torch.utils.data.DataLoader(vidSeqTest, batch_size=testBatchSize, shuffle=False, num_workers=1)
norm_method = Normalize(opt.mean, opt.std) if not opt.no_train: assert opt.train_crop in ['random', 'corner', 'center'] pj_method = PersepectivePj(pj_offsets) if opt.train_crop == 'random': crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size) elif opt.train_crop == 'corner': crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size) elif opt.train_crop == 'center': crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size, crop_positions=['c']) spatial_transform = Compose([ crop_method, RandomHorizontalFlip(), pj_method, ToTensor(opt.norm_value), norm_method ]) temporal_transform = TemporalRandomCrop(opt.sample_duration) target_transform = ClassLabel() training_data = get_training_set(opt, spatial_transform, temporal_transform, target_transform) train_loader = torch.utils.data.DataLoader(training_data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.n_threads, pin_memory=True, drop_last=True) train_logger = Logger(os.path.join(opt.result_path, 'train.log'), ['epoch', 'loss', 'acc', 'lr']) train_batch_logger = Logger(
def main_run(stage, train_data_dir, val_data_dir, stage1Dict, stage1Dict_rgb, stage1Dict_fc, out_dir, seqLen, trainBatchSize, valBatchSize, numEpochs, lr1, decay_factor, decay_step, memSize): #dataset = 'gtea61' num_classes = 61 model_folder = os.path.join( './', out_dir, 'attConvLSTMDoubleResnet', str(seqLen), 'stage' + str(stage)) # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Directory {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files writer = SummaryWriter(model_folder) train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) vid_seq_train = makeDataset(train_data_dir, seqLen=seqLen, fmt='.png', users=['S1', 'S3', 'S4'], spatial_transform=spatial_transform) trainInstances = vid_seq_train.__len__() train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, num_workers=4, pin_memory=True) if val_data_dir is not None: vid_seq_val = makeDataset(val_data_dir, seqLen=seqLen, fmt='.png', users=['S2'], train=False, spatial_transform=Compose([ Scale(256), CenterCrop(224), ToTensor(), normalize ])) valInstances = vid_seq_val.__len__() val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=2, pin_memory=True) train_params = [] model = twoStreamFlowCol(num_classes=num_classes, memSize=memSize, frameModel=stage1Dict_rgb, flowModel=stage1Dict_fc) model.train(False) for params in model.parameters(): params.requires_grad = False model.train(False) train_params = [] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.lstm_cell.parameters(): train_params += [params] params.requires_grad = True for params in model.frameModel.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.frameModel.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.frameModel.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.frameModel.resNet.fc.parameters(): params.requires_grad = True train_params += [params] for params in model.classifier.parameters(): params.requires_grad = True train_params += [params] for params in model.flowModel.lstm_cell.parameters(): train_params += [params] params.requires_grad = True for params in model.flowModel.resNet.layer4[0].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.flowModel.resNet.layer4[0].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.flowModel.resNet.layer4[1].conv1.parameters(): params.requires_grad = True train_params += [params] for params in model.flowModel.resNet.layer4[1].conv2.parameters(): params.requires_grad = True train_params += [params] for params in model.flowModel.resNet.layer4[2].conv1.parameters(): params.requires_grad = True train_params += [params] # for params in model.flowModel.resNet.layer4[2].conv2.parameters(): params.requires_grad = True train_params += [params] # for params in model.flowModel.resNet.fc.parameters(): params.requires_grad = True train_params += [params] model.cuda() trainSamples = vid_seq_train.__len__() min_accuracy = 0 loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.SGD(train_params, lr=lr1, momentum=0.9, weight_decay=5e-4) optim_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_fn, step_size=decay_step, gamma=decay_factor) train_iter = 0 min_accuracy = 0 for epoch in range(numEpochs): epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 model.classifier.train(True) writer.add_scalar('lr', optimizer_fn.param_groups[0]['lr'], epoch + 1) #for i, (inputs, targets) in enumerate(train_loader): for inputs, inputsSN, targets in train_loader: train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariable = Variable(inputs.permute(1, 0, 2, 3, 4).cuda()) inputSNVariable = Variable(inputsSN.permute(1, 0, 2, 3, 4).cuda()) labelVariable = Variable(targets.cuda()) trainSamples += inputs.size(0) output_label, _ = model(inputVariable, inputSNVariable) loss = loss_fn(output_label, labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.cuda()).sum() epoch_loss += loss.item() optim_scheduler.step() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = torch.true_divide(numCorrTrain, trainSamples) * 100 print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format( epoch + 1, avg_loss, trainAccuracy)) writer.add_scalar('train/epoch_loss', avg_loss, epoch + 1) writer.add_scalar('train/accuracy', trainAccuracy, epoch + 1) train_log_loss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) if val_data_dir is not None: model.train(False) val_loss_epoch = 0 val_iter = 0 val_samples = 0 numCorr = 0 with torch.no_grad(): #for j, (inputs, targets) in enumerate(val_loader): for inputs, inputsSN, targets in val_loader: val_iter += 1 val_samples += inputs.size(0) inputVariable = Variable( inputs.permute(1, 0, 2, 3, 4).cuda()) inputSNVariable = Variable( inputsSN.permute(1, 0, 2, 3, 4).cuda()) labelVariable = Variable(targets.cuda(async=True)) #labelVariable = Variable(targets.cuda()) output_label, _ = model(inputVariable, inputSNVariable) val_loss = loss_fn(output_label, labelVariable) val_loss_epoch += val_loss.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.cuda()).sum() val_accuracy = torch.true_divide(numCorr, val_samples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Val: Epoch = {} | Loss {} | Accuracy = {}'.format( epoch + 1, avg_val_loss, val_accuracy)) writer.add_scalar('val/epoch_loss', avg_val_loss, epoch + 1) writer.add_scalar('val/accuracy', val_accuracy, epoch + 1) val_log_loss.write('Val Loss after {} epochs = {}\n'.format( epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format( epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_rgb_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close() writer.export_scalars_to_json(model_folder + "/all_scalars.json") writer.close()
def get_train_utils(opt, model_parameters): assert opt.train_crop in ['random', 'corner', 'center'] spatial_transform = [] if opt.train_crop == 'random': spatial_transform.append( RandomResizedCrop( opt.sample_size, (opt.train_crop_min_scale, 1.0), (opt.train_crop_min_ratio, 1.0 / opt.train_crop_min_ratio))) elif opt.train_crop == 'corner': scales = [1.0] scale_step = 1 / (2**(1 / 4)) for _ in range(1, 5): scales.append(scales[-1] * scale_step) spatial_transform.append(MultiScaleCornerCrop(opt.sample_size, scales)) elif opt.train_crop == 'center': spatial_transform.append(Resize(opt.sample_size)) spatial_transform.append(CenterCrop(opt.sample_size)) normalize = get_normalize_method(opt.mean, opt.std, opt.no_mean_norm, opt.no_std_norm) if not opt.no_hflip: spatial_transform.append(RandomHorizontalFlip()) if opt.colorjitter: spatial_transform.append(ColorJitter()) spatial_transform.append(ToTensor()) if opt.input_type == 'flow': spatial_transform.append(PickFirstChannels(n=2)) spatial_transform.append(ScaleValue(opt.value_scale)) spatial_transform.append(normalize) spatial_transform = Compose(spatial_transform) assert opt.train_t_crop in ['random', 'center'] temporal_transform = [] if opt.sample_t_stride > 1: temporal_transform.append(TemporalSubsampling(opt.sample_t_stride)) if opt.train_t_crop == 'random': temporal_transform.append(TemporalRandomCrop(opt.sample_duration)) elif opt.train_t_crop == 'center': temporal_transform.append(TemporalCenterCrop(opt.sample_duration)) temporal_transform = TemporalCompose(temporal_transform) train_data = get_training_data(opt.video_path, opt.annotation_path, opt.dataset, opt.input_type, opt.file_type, spatial_transform, temporal_transform) if opt.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( train_data) else: train_sampler = None train_loader = torch.utils.data.DataLoader(train_data, batch_size=opt.batch_size, shuffle=(train_sampler is None), num_workers=opt.n_threads, pin_memory=True, sampler=train_sampler, worker_init_fn=worker_init_fn) if opt.is_master_node: train_logger = Logger(opt.result_path / 'train.log', ['epoch', 'loss', 'acc', 'lr']) train_batch_logger = Logger( opt.result_path / 'train_batch.log', ['epoch', 'batch', 'iter', 'loss', 'acc', 'lr']) else: train_logger = None train_batch_logger = None if opt.nesterov: dampening = 0 else: dampening = opt.dampening optimizer = SGD(model_parameters, lr=opt.learning_rate, momentum=opt.momentum, dampening=dampening, weight_decay=opt.weight_decay, nesterov=opt.nesterov) assert opt.lr_scheduler in ['plateau', 'multistep'] assert not (opt.lr_scheduler == 'plateau' and opt.no_val) if opt.lr_scheduler == 'plateau': scheduler = lr_scheduler.ReduceLROnPlateau( optimizer, 'min', patience=opt.plateau_patience) else: scheduler = lr_scheduler.MultiStepLR(optimizer, opt.multistep_milestones) return (train_loader, train_sampler, train_logger, train_batch_logger, optimizer, scheduler)
def main_run(dataset, trainDir, valDir, outDir, stackSize, trainBatchSize, valBatchSize, numEpochs, lr1, decay_factor, decay_step, uniform_sampling, debug): # GTEA 61 num_classes = 61 # Train/Validation/Test split train_splits = ["S1", "S3", "S4"] val_splits = ["S2"] if debug: n_workers = 0 device = 'cpu' else: n_workers = 4 device = 'cuda' min_accuracy = 0 model_folder = os.path.join('./', outDir, dataset, 'flow') # Dir for saving models and log files # Create the dir if os.path.exists(model_folder): print('Dir {} exists!'.format(model_folder)) sys.exit() os.makedirs(model_folder) # Log files train_log_loss = open((model_folder + '/train_log_loss.txt'), 'w') train_log_acc = open((model_folder + '/train_log_acc.txt'), 'w') val_log_loss = open((model_folder + '/val_log_loss.txt'), 'w') val_log_acc = open((model_folder + '/val_log_acc.txt'), 'w') #num_workers = 4 # Data loader normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) vid_seq_train = makeDataset(trainDir, train_splits, spatial_transform=spatial_transform, sequence=False, stackSize=stackSize, fmt='.png', uniform_sampling=uniform_sampling) train_loader = torch.utils.data.DataLoader(vid_seq_train, batch_size=trainBatchSize, shuffle=True, sampler=None, num_workers=n_workers, pin_memory=True) vid_seq_val = makeDataset(trainDir, val_splits, spatial_transform=Compose([ Scale(256), CenterCrop(224), ToTensor(), normalize ]), sequence=False, stackSize=stackSize, fmt='.png', phase='Test', uniform_sampling=uniform_sampling) val_loader = torch.utils.data.DataLoader(vid_seq_val, batch_size=valBatchSize, shuffle=False, num_workers=n_workers, pin_memory=True) valInstances = vid_seq_val.__len__() trainInstances = vid_seq_train.__len__() print('Number of samples in the dataset: training = {} | validation = {}'. format(trainInstances, valInstances)) model = flow_resnet34(True, channels=2 * stackSize, num_classes=num_classes) model.train(True) train_params = list(model.parameters()) model.to(device) loss_fn = nn.CrossEntropyLoss() optimizer_fn = torch.optim.SGD(train_params, lr=lr1, momentum=0.9, weight_decay=5e-4) optim_scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer_fn, milestones=decay_step, gamma=decay_factor) train_iter = 0 for epoch in range(numEpochs): epoch_loss = 0 numCorrTrain = 0 trainSamples = 0 iterPerEpoch = 0 model.train(True) for i, (inputs, targets) in enumerate(train_loader): train_iter += 1 iterPerEpoch += 1 optimizer_fn.zero_grad() inputVariable = inputs.to(device) labelVariable = targets.to(device) trainSamples += inputs.size(0) output_label, _ = model(inputVariable) loss = loss_fn(output_label, labelVariable) loss.backward() optimizer_fn.step() _, predicted = torch.max(output_label.data, 1) numCorrTrain += (predicted == targets.to(device)).sum() epoch_loss += loss.data.item() optim_scheduler.step() avg_loss = epoch_loss / iterPerEpoch trainAccuracy = (numCorrTrain.data.item() / trainSamples) * 100 print('Train: Epoch = {} | Loss = {} | Accuracy = {}'.format( epoch + 1, avg_loss, trainAccuracy)) train_log_loss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avg_loss)) train_log_acc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) if (epoch + 1) % 1 == 0: model.train(False) val_loss_epoch = 0 val_iter = 0 val_samples = 0 numCorr = 0 for j, (inputs, targets) in enumerate(val_loader): val_iter += 1 val_samples += inputs.size(0) inputVariable = inputs.to(device) labelVariable = targets.to(device) output_label, _ = model(inputVariable) val_loss = loss_fn(output_label, labelVariable) val_loss_epoch += val_loss.data.item() _, predicted = torch.max(output_label.data, 1) numCorr += (predicted == targets.to(device)).sum() val_accuracy = (numCorr.data.item() / val_samples) * 100 avg_val_loss = val_loss_epoch / val_iter print('Validation: Epoch = {} | Loss = {} | Accuracy = {}'.format( epoch + 1, avg_val_loss, val_accuracy)) val_log_loss.write('Val Loss after {} epochs = {}\n'.format( epoch + 1, avg_val_loss)) val_log_acc.write('Val Accuracy after {} epochs = {}%\n'.format( epoch + 1, val_accuracy)) if val_accuracy > min_accuracy: save_path_model = (model_folder + '/model_flow_state_dict.pth') torch.save(model.state_dict(), save_path_model) min_accuracy = val_accuracy else: if (epoch + 1) % 10 == 0: save_path_model = (model_folder + '/model_flow_state_dict_epoch' + str(epoch + 1) + '.pth') torch.save(model.state_dict(), save_path_model) train_log_loss.close() train_log_acc.close() val_log_acc.close() val_log_loss.close()
def main_run(numEpochs, lr, stepSize, decayRate, trainBatchSize, seqLen, memSize, evalInterval, evalMode, numWorkers, outDir, fightsDir_train, noFightsDir_train, fightsDir_test, noFightsDir_test): train_dataset_dir_fights = fightsDir_train train_dataset_dir_noFights = noFightsDir_train test_dataset_dir_fights = fightsDir_test test_dataset_dir_noFights = noFightsDir_test trainDataset, trainLabels, trainNumFrames = make_split( train_dataset_dir_fights, train_dataset_dir_noFights) testDataset, testLabels, testNumFrames = make_split( test_dataset_dir_fights, test_dataset_dir_noFights) mean = [0.485, 0.456, 0.406] std = [0.229, 0.224, 0.225] normalize = Normalize(mean=mean, std=std) spatial_transform = Compose([ Scale(256), RandomHorizontalFlip(), MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224), ToTensor(), normalize ]) vidSeqTrain = VideoDataset(trainDataset, trainLabels, trainNumFrames, spatial_transform=spatial_transform, seqLen=seqLen) trainLoader = torch.utils.data.DataLoader(vidSeqTrain, batch_size=trainBatchSize, shuffle=True, num_workers=numWorkers, pin_memory=True, drop_last=True) if evalMode == 'centerCrop': test_spatial_transform = Compose( [Scale(256), CenterCrop(224), ToTensor(), normalize]) testBatchSize = 1 elif evalMode == 'tenCrops': test_spatial_transform = Compose( [Scale(256), TenCrops(size=224, mean=mean, std=std)]) testBatchSize = 1 elif evalMode == 'fiveCrops': test_spatial_transform = Compose( [Scale(256), FiveCrops(size=224, mean=mean, std=std)]) testBatchSize = 1 elif evalMode == 'horFlip': test_spatial_transform = Compose([ Scale(256), CenterCrop(224), FlippedImagesTest(mean=mean, std=std) ]) testBatchSize = 1 vidSeqTest = VideoDataset(testDataset, testLabels, testNumFrames, seqLen=seqLen, spatial_transform=test_spatial_transform) testLoader = torch.utils.data.DataLoader(vidSeqTest, batch_size=testBatchSize, shuffle=False, num_workers=int(numWorkers / 2), pin_memory=True) numTrainInstances = vidSeqTrain.__len__() numTestInstances = vidSeqTest.__len__() print('Number of training samples = {}'.format(numTrainInstances)) print('Number of testing samples = {}'.format(numTestInstances)) modelFolder = './experiments_' + outDir # Dir for saving models and log files # Create the dir if os.path.exists(modelFolder): print(modelFolder + ' exists!!!') sys.exit() else: os.makedirs(modelFolder) # Log files writer = SummaryWriter(modelFolder) trainLogLoss = open((modelFolder + '/trainLogLoss.txt'), 'w') trainLogAcc = open((modelFolder + '/trainLogAcc.txt'), 'w') testLogLoss = open((modelFolder + '/testLogLoss.txt'), 'w') testLogAcc = open((modelFolder + '/testLogAcc.txt'), 'w') model = ViolenceModel(mem_size=memSize) trainParams = [] for params in model.parameters(): params.requires_grad = True trainParams += [params] model.train(True) model.cuda() lossFn = nn.CrossEntropyLoss() optimizerFn = torch.optim.RMSprop(trainParams, lr=lr) optimScheduler = torch.optim.lr_scheduler.StepLR(optimizerFn, stepSize, decayRate) minAccuracy = 50 for epoch in range(numEpochs): optimScheduler.step() epochLoss = 0 numCorrTrain = 0 iterPerEpoch = 0 model.train(True) print('Epoch = {}'.format(epoch + 1)) writer.add_scalar('lr', optimizerFn.param_groups[0]['lr'], epoch + 1) for i, (inputs, targets) in enumerate(trainLoader): iterPerEpoch += 1 optimizerFn.zero_grad() inputVariable1 = Variable(inputs.permute(1, 0, 2, 3, 4).cuda()) labelVariable = Variable(targets.cuda()) outputLabel = model(inputVariable1) loss = lossFn(outputLabel, labelVariable) loss.backward() optimizerFn.step() outputProb = torch.nn.Softmax(dim=1)(outputLabel) _, predicted = torch.max(outputProb.data, 1) numCorrTrain += (predicted == targets.cuda()).sum() epochLoss += loss.data[0] avgLoss = epochLoss / iterPerEpoch trainAccuracy = (numCorrTrain / numTrainInstances) * 100 print('Training: Loss = {} | Accuracy = {}% '.format( avgLoss, trainAccuracy)) writer.add_scalar('train/epochLoss', avgLoss, epoch + 1) writer.add_scalar('train/accuracy', trainAccuracy, epoch + 1) trainLogLoss.write('Training loss after {} epoch = {}\n'.format( epoch + 1, avgLoss)) trainLogAcc.write('Training accuracy after {} epoch = {}\n'.format( epoch + 1, trainAccuracy)) if (epoch + 1) % evalInterval == 0: model.train(False) print('Evaluating...') testLossEpoch = 0 testIter = 0 numCorrTest = 0 for j, (inputs, targets) in enumerate(testLoader): testIter += 1 if evalMode == 'centerCrop': inputVariable1 = Variable(inputs.permute(1, 0, 2, 3, 4).cuda(), volatile=True) else: inputVariable1 = Variable(inputs[0].cuda(), volatile=True) labelVariable = Variable(targets.cuda(async=True), volatile=True) outputLabel = model(inputVariable1) outputLabel_mean = torch.mean(outputLabel, 0, True) testLoss = lossFn(outputLabel_mean, labelVariable) testLossEpoch += testLoss.data[0] _, predicted = torch.max(outputLabel_mean.data, 1) numCorrTest += (predicted == targets[0]).sum() testAccuracy = (numCorrTest / numTestInstances) * 100 avgTestLoss = testLossEpoch / testIter print('Testing: Loss = {} | Accuracy = {}% '.format( avgTestLoss, testAccuracy)) writer.add_scalar('test/epochloss', avgTestLoss, epoch + 1) writer.add_scalar('test/accuracy', testAccuracy, epoch + 1) testLogLoss.write('Test Loss after {} epochs = {}\n'.format( epoch + 1, avgTestLoss)) testLogAcc.write('Test Accuracy after {} epochs = {}%\n'.format( epoch + 1, testAccuracy)) if testAccuracy > minAccuracy: savePathClassifier = (modelFolder + '/bestModel.pth') torch.save(model, savePathClassifier) minAccuracy = testAccuracy trainLogAcc.close() testLogAcc.close() trainLogLoss.close() testLogLoss.close() writer.export_scalars_to_json(modelFolder + "/all_scalars.json") writer.close() return True
def main(): resnet_in = generate_model(opt) resnet_in.module.fc = Identity() model = ReNet34(resnet_in, encode_length=encode_length) if opt.no_mean_norm and not opt.std_norm: norm_method = Normalize([0, 0, 0], [1, 1, 1]) elif not opt.std_norm: norm_method = Normalize(opt.mean, [1, 1, 1]) else: norm_method = Normalize(opt.mean, opt.std) if not opt.no_train: assert opt.train_crop in ['random', 'corner', 'center'] if opt.train_crop == 'random': crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size) elif opt.train_crop == 'corner': crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size) elif opt.train_crop == 'center': crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size, crop_positions=['c']) ## train loader spatial_transform = Compose([ crop_method, RandomHorizontalFlip(), ToTensor(opt.norm_value), norm_method ]) temporal_transform = TemporalRandomCrop(opt.sample_duration) target_transform = ClassLabel() training_data = get_training_set(opt, spatial_transform, temporal_transform, target_transform) train_loader = torch.utils.data.DataLoader(training_data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.n_threads, pin_memory=True) ## test loader spatial_transform = Compose([ Scale(int(opt.sample_size / opt.scale_in_test)), CornerCrop(opt.sample_size, opt.crop_position_in_test), ToTensor(opt.norm_value), norm_method ]) temporal_transform = LoopPadding(opt.sample_duration) target_transform = ClassLabel() test_data = get_test_set(opt, spatial_transform, temporal_transform, target_transform) test_loader = torch.utils.data.DataLoader(test_data, batch_size=opt.batch_size, shuffle=False, num_workers=opt.n_threads, pin_memory=True) ## Database loader spatial_transform = Compose([ Scale(int(opt.sample_size / opt.scale_in_test)), CornerCrop(opt.sample_size, opt.crop_position_in_test), ToTensor(opt.norm_value), norm_method ]) temporal_transform = LoopPadding(opt.sample_duration) target_transform = ClassLabel() validation_data = get_validation_set(opt, spatial_transform, temporal_transform, target_transform) database_loader = torch.utils.data.DataLoader( validation_data, batch_size=opt.batch_size, shuffle=False, num_workers=opt.n_threads, pin_memory=True) if opt.nesterov: dampening = 0 else: dampening = opt.dampening optimizer = optim.SGD(model.parameters(), lr=opt.learning_rate, momentum=opt.momentum, dampening=dampening, weight_decay=opt.weight_decay, nesterov=opt.nesterov) scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=opt.lr_patience) if opt.resume_path: print('loading checkpoint {}'.format(opt.resume_path)) checkpoint = torch.load(opt.resume_path) assert opt.arch == checkpoint['arch'] opt.begin_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) if not opt.no_train: optimizer.load_state_dict(checkpoint['optimizer']) for state in optimizer.state.values(): for k, v in state.items(): if torch.is_tensor(v): state[k] = v.cuda() print('run') for epoch in range(opt.begin_epoch, opt.n_epochs + 1): model.cuda().train() for i, (images, labels) in enumerate(train_loader): images = Variable(images.cuda()) labels = Variable(labels.cuda().long()) # Forward + Backward + Optimize optimizer.zero_grad() x, _, b = model(images) target_b = F.cosine_similarity(b[:int(labels.size(0) / 2)], b[int(labels.size(0) / 2):]) target_x = F.cosine_similarity(x[:int(labels.size(0) / 2)], x[int(labels.size(0) / 2):]) loss = F.mse_loss(target_b, target_x) loss.backward() optimizer.step() scheduler.step() # Test the Model if (epoch + 1) % 10 == 0: model.eval() retrievalB, retrievalL, queryB, queryL = compress( database_loader, test_loader, model) result_map = calculate_top_map(qB=queryB, rB=retrievalB, queryL=queryL, retrievalL=retrievalL, topk=100) print('--------mAP@100: {}--------'.format(result_map))