def test(dataset="UCF-101"): dataset = dataset.lower() with tf.Graph().as_default(): images_placeholder, labels_placeholder = placeholder_inputs() c3d = C3D.C3D(num_class=101) logit = c3d.model(images_placeholder, batch_size=BATCH_SIZE) correct_pred = tf.reduce_sum( tf.cast(tf.equal(tf.argmax(logit, 1), labels_placeholder), tf.int16)) print(correct_pred) sess = tf.Session() sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.restore(sess, model_save_dir + '/' + dataset + '.model') test_data = dataReader.dataReader("data/" + dataset + "_test_list.txt", batch_size=BATCH_SIZE) finished = False cnt = 0 total = 0 while (not finished): test_images, test_labels = test_data.get_next_batch() finished = test_data.is_finished() _correct_pred = sess.run([correct_pred], feed_dict={ images_placeholder: test_images, labels_placeholder: test_labels }) total += test_images.shape[0] cnt += _correct_pred[0] print(cnt, total, cnt / float(total))
def main(args): # Create model directory for saving trained models if not os.path.exists(args.model_path): os.makedirs(args.model_path) transform = None # Build data loader data_loader = get_loader(root=args.video_root, list_file=args.list_file, transform=transform, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) # Build the models model = C3D(args.num_classes).to(device) # Loss and optimizer criterion = nn.CrossEntropyLoss() params = model.parameters() optimizer = torch.optim.Adam(params, lr=args.learning_rate) # Train the models total_step = len(data_loader) for epoch in range(args.num_epochs): for i, sample in enumerate(data_loader): # Set mini-batch dataset clip, target = sample clip = clip.to(device) target = target.to(device) # clip = sample['clip'].to(device) # target = sample['label'].to(device) # Forward, backward and optimize outputs = model(clip) # preds = torch.max(outputs, 1)[1] loss = criterion(outputs, target) # zero the parameter gradients optimizer.zero_grad() loss.backward() optimizer.step() # Print log info if i % args.log_step == 0: print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format( epoch, args.num_epochs, i, total_step, loss.item())) # Save the model checkpoints if (i + 1) % args.save_step == 0: torch.save( model.state_dict(), os.path.join(args.model_path, 'c3d-{}-{}.ckpt'.format(epoch + 1, i + 1)))
def predict(): X = get_sport_clip('test') X = Variable(X) X = X.cuda() # get network pretrained model net = C3D() net.load_state_dict(torch.load('c3d.pickle')) net.cuda() net.eval() # perform prediction prediction = net(X) prediction = prediction.data.cpu().numpy() # read labels labels = read_labels_from_file('labels.txt') # print top predictions top_inds = prediction[0].argsort()[::-1][:5] # reverse sort and take five largest items print('\nTop 5:') for i in top_inds: print('{:.5f} {}'.format(prediction[0][i], labels[i]))
def init_model(): # init model return C3D(num_classes=101, pretrained=True)
def main(config): if config.model == 'c3d': model, params = C3D(config) elif config.model == 'convlstm': model, params = ConvLSTM(config) elif config.model == 'densenet': model, params = densenet(config) elif config.model == 'densenet_lean': model, params = densenet_lean(config) elif config.model == 'resnext': model, params = resnext(config) else: model, params = densenet_lean(config) dataset = config.dataset sample_size = config.sample_size stride = config.stride sample_duration = config.sample_duration cv = config.num_cv # crop_method = GroupRandomScaleCenterCrop(size=sample_size) crop_method = MultiScaleRandomCrop(config.scales, config.sample_size[0]) # norm = Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) norm = Normalize([114.7748, 107.7354, 99.475], [1, 1, 1]) # spatial_transform = Compose( # [crop_method, # GroupRandomHorizontalFlip(), # ToTensor(1), norm]) spatial_transform = Compose([ RandomHorizontalFlip(), crop_method, ToTensor(config.norm_value), norm ]) # temporal_transform = RandomCrop(size=sample_duration, stride=stride) temporal_transform = TemporalRandomCrop(config.sample_duration, config.downsample) target_transform = Label() train_batch = config.train_batch train_data = RWF2000('/content/RWF_2000/frames/', g_path + '/RWF-2000.json', 'training', spatial_transform, temporal_transform, target_transform, dataset) train_loader = DataLoader(train_data, batch_size=train_batch, shuffle=True, num_workers=4, pin_memory=True) crop_method = GroupScaleCenterCrop(size=sample_size) norm = Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) spatial_transform = Compose([crop_method, ToTensor(), norm]) temporal_transform = CenterCrop(size=sample_duration, stride=stride) target_transform = Label() val_batch = config.val_batch val_data = RWF2000('/content/RWF_2000/frames/', g_path + '/RWF-2000.json', 'validation', spatial_transform, temporal_transform, target_transform, dataset) val_loader = DataLoader(val_data, batch_size=val_batch, shuffle=False, num_workers=4, pin_memory=True) if not os.path.exists('{}/pth'.format(config.output)): os.mkdir('{}/pth'.format(config.output)) if not os.path.exists('{}/log'.format(config.output)): os.mkdir('{}/log'.format(config.output)) batch_log = Log( '{}/log/{}_fps{}_{}_batch{}.log'.format( config.output, config.model, sample_duration, dataset, cv, ), ['epoch', 'batch', 'iter', 'loss', 'acc', 'lr']) epoch_log = Log( '{}/log/{}_fps{}_{}_epoch{}.log'.format(config.output, config.model, sample_duration, dataset, cv), ['epoch', 'loss', 'acc', 'lr']) val_log = Log( '{}/log/{}_fps{}_{}_val{}.log'.format(config.output, config.model, sample_duration, dataset, cv), ['epoch', 'loss', 'acc']) criterion = nn.CrossEntropyLoss().to(device) # criterion = nn.BCELoss().to(device) learning_rate = config.learning_rate momentum = config.momentum weight_decay = config.weight_decay optimizer = torch.optim.SGD(params=params, lr=learning_rate, momentum=momentum, weight_decay=weight_decay, dampening=False, nesterov=False) # optimizer = torch.optim.Adam(params=params, lr = learning_rate, weight_decay= weight_decay) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer, verbose=True, factor=config.factor, min_lr=config.min_lr) acc_baseline = config.acc_baseline loss_baseline = 1 for p in range(1, config.num_prune): if p > 0: model = torch.load('{}/pth/prune_{}.pth'.format( config.output, p - 1)) print(f"Prune {p}/{config.num_prune}") params = sum([np.prod(p.size()) for p in model.parameters()]) print("Number of Parameters: %.1fM" % (params / 1e6)) model = prune_model(model) params = sum([np.prod(p.size()) for p in model.parameters()]) print("Number of Parameters: %.1fM" % (params / 1e6)) model.to(config.device) acc_baseline = 0 for i in range(5): train(i, train_loader, model, criterion, optimizer, device, batch_log, epoch_log) val_loss, val_acc = val(i, val_loader, model, criterion, device, val_log) scheduler.step(val_loss) if val_acc > acc_baseline or (val_acc >= acc_baseline and val_loss < loss_baseline): # torch.save( # model.state_dict(), # '{}/pth/prune_{}_{}_fps{}_{}{}_{}_{:.4f}_{:.6f}.pth'.format( # config.output, p, config.model, sample_duration, dataset, cv, i, val_acc, # val_loss)) torch.save(model, '{}/pth/prune_{}.pth'.format(config.output, p))
def train(dataset="UCF-101"): # Get the sets of images and labels for training, validation, and # Tell TensorFlow that the model will be built into the default Graph. dataset = dataset.lower() # Create model directory if not os.path.exists(model_save_dir): os.makedirs(model_save_dir) # use_pretrained_model = True # model_filename = "./sports1m_finetuning_ucf101.model" with tf.Graph().as_default(): images_placeholder, labels_placeholder = placeholder_inputs() lr = tf.placeholder(tf.float32) c3d = C3D.C3D(num_class=101) logit = c3d.model(images_placeholder, batch_size=BATCH_SIZE) loss, accuracy, pred = get_loss_accuracy(logit, labels_placeholder) train_op = tf.train.AdamOptimizer(lr).minimize(loss) train_data = dataReader.dataReader("data/" + dataset + "_train_list.txt", batch_size=BATCH_SIZE) test_data = dataReader.dataReader("data/" + dataset + "_test_list.txt", batch_size=BATCH_SIZE) init = tf.global_variables_initializer() sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) sess.run(init) saver = tf.train.Saver() # ckpt=tf.train.get_checkpoint_state(model_save_dir) # if ckpt and ckpt.model_checkpoint_path: # print(ckpt.model_checkpoint_path) # saver.restore(sess, ckpt.model_checkpoint_path) # Create a session for running Ops on the Graph. learning_rate = 0.003 for step in range(FLAGS.max_steps): if step != 0 and step % 2e+5 == 0: learning_rate /= 10 # start_time = time.time() train_images, train_labels = train_data.get_next_batch() sess.run(train_op, feed_dict={ images_placeholder: train_images, labels_placeholder: train_labels, lr: learning_rate }) # duration = time.time() - start_time # print('Step %d: %.3f sec' % (step, duration)) # Save a checkpoint and evaluate the model periodically. if (step) % 10 == 0 or (step + 1) == FLAGS.max_steps: print('step %d Train' % (step)) acc, _loss, _pred, _logit = sess.run( [accuracy, loss, pred, logit], feed_dict={ images_placeholder: train_images, labels_placeholder: train_labels, lr: learning_rate }) print(_logit) # print (_pred) print("accuracy: " + "{:.5f}".format(acc) + " loss:" + "{:.5f}".format(_loss)) if (step) % 100 == 0 or (step + 1) == FLAGS.max_steps: print('step %d Test' % (step)) test_images, test_labels = test_data.get_next_batch() _loss, acc = sess.run( [loss, accuracy], feed_dict={ images_placeholder: test_images, labels_placeholder: test_labels, lr: learning_rate }) print("accuracy: " + "{:.5f}".format(acc) + " loss:" + "{:.5f}".format(_loss)) saver.save(sess, model_save_dir + '/' + dataset + '.model') print("done")
# Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, sampler=train_sampler, num_workers=8) validation_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, sampler=valid_sampler, num_workers=8) #creating network net = C3D() if args.resume: print('==> Resuming from checkpoint..') checkpoint = torch.load('./checkpoint/ckpt.pth') net.load_state_dict(checkpoint['net']) best_loss = checkpoint['loss'] start_epoch = checkpoint['epoch'] net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count())) net.cuda() criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9,
def fine_tune(load_dir, save_dir, filename, dataloader_train, dataloader_train_sampler, dataloader_val): # load pretrained model pretrained_weights = torch.load(os.path.join(load_dir, filename), map_location='cpu') feature_state_dict = {key: value for key, value in pretrained_weights.items() if key not in ['fc8.weight', 'fc8.bias']} net = C3D.C3D() state_dict = deepcopy(net.state_dict()) state_dict.update(feature_state_dict) net.load_state_dict(state_dict) net.to(device) net.train() # pdb.set_trace() summary(net, (3,args.n_frames_per_clip,112,112)) # pdb.set_trace() num_epochs = 100 step = 0 # determine optimizer criterion = nn.CrossEntropyLoss() fc_lr_layers = list(map(id, net.fc8.parameters())) pretrained_lr_layers = [p for p in net.parameters() if id(p) not in fc_lr_layers and p.requires_grad==True] # pretrained_lr_layers = filter(lambda p: # id(p) not in fc_lr_layers, model.parameters()) optimizer = torch.optim.SGD([ {"params": net.fc8.parameters()}, {"params": pretrained_lr_layers, "lr": 1e-4, 'weight_decay':5e-3} ], lr=1e-3, momentum=0.9, weight_decay=1e-2) # optimizer = torch.optim.SGD(net.fc8.parameters(), lr=1e-3, momentum=0.9, weight_decay=1e-5) # optimizer = torch.optim.Adam([ # {"params": model.module.fc.parameters()}, # {"params": pretrained_lr_layers, "lr": 1e-4} # ], lr=1e-3) # optimizer = optim.SGD(model.parameters(), lr=0.001,momentum=0.9, weight_decay=5e-4) train_logger = utils.Logger(os.path.join('output', 'C3D-fine-tune-all.log'), ['step', 'train_loss', 'train_acc', 'val_loss', 'val_acc', 'lr_feature', 'lr_fc']) # scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', factor=0.5, patience=1) scheduler = lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.1) train_loss = utils.AverageMeter() train_acc = utils.AverageMeter() val_loss = utils.AverageMeter() val_acc = utils.AverageMeter() batch_cache = 1 for epoch in trange(num_epochs): # loop over the dataset multiple times train_loss.reset() train_acc.reset() for data in dataloader_train: outputs, logits, labels = forward(net, data) # print(batch_cache) # pdb.set_trace() # if batch_cache == 8: # accumulate number of batches because gpu memory limits the batch size # optimizer.step() # optimizer.zero_grad() loss_ = criterion(logits, labels) optimizer.zero_grad() loss_.backward() optimizer.step() train_loss.update(loss_.item()) train_acc.update(utils.calculate_accuracy(outputs, labels)) # if batch_cache == 8: # batch_cache = 0 if step % 100 == 0: val_loss.reset() val_acc.reset() net.eval() # for data_sampler in dataloader_train_sampler: # outputs_sampler, logits_sampler, labels_sampler = forward(net, data_sampler) # loss_sampler_ = criterion(logits_sampler, labels_sampler) # train_loss.update(loss_sampler_.item()) # train_acc.update(utils.calculate_accuracy(outputs_sampler, labels_sampler)) for data_val in dataloader_val: outputs_val, logits_val, labels_val = forward(net, data_val) val_loss_ = criterion(logits_val, labels_val) val_loss.update(val_loss_.item()) val_acc.update(utils.calculate_accuracy(outputs_val, labels_val)) net.train() print('epoch{}/{} train_acc:{:.3f} train_loss:{:.3f} val_acc:{:.3f} val_loss:{:.3f}'.format( epoch + 1, num_epochs, train_acc.val, train_loss.val, val_acc.avg, val_loss.avg )) train_logger.log({ 'step': step, 'train_loss': train_loss.val, 'train_acc': train_acc.val, 'val_loss': val_loss.avg, 'val_acc': val_acc.avg, # 'lr_feature': optimizer.param_groups[1]['lr'], 'lr_feature': 0, 'lr_fc': optimizer.param_groups[0]['lr'] }) if step % 200 == 0: utils.save_checkpoint(net, optimizer, step, save_dir, 'c3d.pth') step += 1 # batch_cache += 1 scheduler.step()
def train_model(dataset=dataset, save_dir=save_dir, num_classes=num_classes, lr=lr, num_epochs=nEpochs): """ Args: num_classes (int): Number of classes in the data num_epochs (int, optional): Number of epochs to train for. """ model = C3D(num_classes=num_classes, pretrained=True) train_params = [{ 'params': C3D.get_1x_lr_params(model), 'lr': lr }, { 'params': C3D.get_10x_lr_params(model), 'lr': lr * 10 }] criterion = nn.CrossEntropyLoss( ) # standard crossentropy loss for classification optimizer = optim.SGD(train_params, lr=lr, momentum=0.9, weight_decay=5e-4) scheduler = optim.lr_scheduler.StepLR( optimizer, step_size=10, gamma=0.1) # the scheduler divides the lr by 10 every 10 epochs # if resume_epoch == 0: # print("Training {} from scratch...".format(modelName)) # else: # checkpoint = torch.load(os.path.join(save_dir, 'models', saveName + '_epoch-' + str(resume_epoch - 1) + '.pth.tar'), # map_location=lambda storage, loc: storage) # Load all tensors onto the CPU # print("Initializing weights from: {}...".format( # os.path.join(save_dir, 'models', saveName + '_epoch-' + str(resume_epoch - 1) + '.pth.tar'))) # model.load_state_dict(checkpoint['state_dict']) # optimizer.load_state_dict(checkpoint['opt_dict']) # print('Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) model.to(device) criterion.to(device) # log_dir = os.path.join(save_dir, 'models', datetime.now().strftime('%b%d_%H-%M-%S') + '_' + socket.gethostname()) print('Training model on {} dataset...'.format(dataset)) train_dataloader = DataLoader(VideoDataset(dataset=dataset, split='train', clip_len=16), batch_size=20, shuffle=True, num_workers=4) val_dataloader = DataLoader(VideoDataset(dataset=dataset, split='val', clip_len=16), batch_size=20, num_workers=4) test_dataloader = DataLoader(VideoDataset(dataset=dataset, split='test', clip_len=16), batch_size=20, num_workers=4) trainval_loaders = {'train': train_dataloader, 'val': val_dataloader} trainval_sizes = { x: len(trainval_loaders[x].dataset) for x in ['train', 'val'] } test_size = len(test_dataloader.dataset) train_acc_list, train_loss_list, val_acc_list, val_loss_list = [], [], [], [] for epoch in range(num_epochs): # each epoch has a training and validation step for phase in ['train', 'val']: # start_time = timeit.default_timer() # reset the running loss and corrects running_loss = 0.0 running_corrects = 0.0 # set model to train() or eval() mode depending on whether it is trained # or being validated. Primarily affects layers such as BatchNorm or Dropout. if phase == 'train': # scheduler.step() is to be called once every epoch during training scheduler.step() model.train() else: model.eval() for inputs, labels in tqdm(trainval_loaders[phase]): # move inputs and labels to the device the training is taking place on inputs = Variable(inputs, requires_grad=True).to(device) labels = Variable(labels).long().to(device) optimizer.zero_grad() if phase == 'train': outputs = model(inputs) else: with torch.no_grad(): outputs = model(inputs) probs = nn.Softmax(dim=1)(outputs) preds = torch.max(probs, 1)[1] loss = criterion(outputs, labels) if phase == 'train': loss.backward() optimizer.step() running_loss += loss.item() * inputs.size(0) running_corrects += torch.sum(preds == labels.data) epoch_loss = running_loss / trainval_sizes[phase] epoch_acc = running_corrects.double() / trainval_sizes[phase] if phase == 'train': train_acc_list.append(epoch_acc) train_loss_list.append(epoch_loss) else: val_acc_list.append(epoch_acc) val_loss_list.append(epoch_loss) # if phase == 'train': # writer.add_scalar('data/train_loss_epoch', epoch_loss, epoch) # writer.add_scalar('data/train_acc_epoch', epoch_acc, epoch) # else: # writer.add_scalar('data/val_loss_epoch', epoch_loss, epoch) # writer.add_scalar('data/val_acc_epoch', epoch_acc, epoch) print("[{}] Epoch: {}/{} Loss: {} Acc: {}".format( phase, epoch + 1, nEpochs, epoch_loss, epoch_acc)) if phase == 'val': torch.save( model, '{}model_epoch{}_accuracy_{}.pkl'.format( path, epoch + 1, epoch_acc)) # stop_time = timeit.default_timer() # print("Execution time: " + str(stop_time - start_time) + "\n") # # if epoch % save_epoch == (save_epoch - 1): # torch.save({ # 'epoch': epoch + 1, # 'state_dict': model.state_dict(), # 'opt_dict': optimizer.state_dict(), # }, os.path.join(save_dir, 'models', saveName + '_epoch-' + str(epoch) + '.pth.tar')) # print("Save model at {}\n".format(os.path.join(save_dir, 'models', saveName + '_epoch-' + str(epoch) + '.pth.tar'))) # if useTest and epoch % test_interval == (test_interval - 1): # model.eval() # start_time = timeit.default_timer() # # running_loss = 0.0 # running_corrects = 0.0 # # for inputs, labels in tqdm(test_dataloader): # inputs = inputs.to(device) # labels = labels.to(device) # # with torch.no_grad(): # outputs = model(inputs) # probs = nn.Softmax(dim=1)(outputs) # preds = torch.max(probs, 1)[1] # loss = criterion(outputs, labels) # # running_loss += loss.item() * inputs.size(0) # running_corrects += torch.sum(preds == labels.data) # # epoch_loss = running_loss / test_size # epoch_acc = running_corrects.double() / test_size # # writer.add_scalar('data/test_loss_epoch', epoch_loss, epoch) # writer.add_scalar('data/test_acc_epoch', epoch_acc, epoch) # # print("[test] Epoch: {}/{} Loss: {} Acc: {}".format(epoch+1, nEpochs, epoch_loss, epoch_acc)) # stop_time = timeit.default_timer() # print("Execution time: " + str(stop_time - start_time) + "\n") # # writer.close() return train_acc_list, train_loss_list, val_acc_list, val_loss_list
print('video_path: ', opt.video_path) print('test_data_path: ', test_data_path) print('annotation_test_path: ', opt.annotation_test_path) print('meta_test_path: ', opt.meta_test_path) print('log_path: ', opt.log_path) print('log_test_path: ', opt.log_test_path) print('pretrained_model_path:', opt.pretrained_model_path) test_dataset = Dataset(test_data_path, opt.annotation_path, opt.labels_path) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=16, pin_memory=True) model = C3D() device = opt.device if opt.is_parallel: opt.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") if torch.cuda.device_count() > 1: model = nn.DataParallel(model) print("multi GPUs: ", torch.cuda.device_count()) elif device != "": opt.device = "cuda:" + str(device) else: opt.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") if opt.pretrained_model_path != '': print("load from ", opt.pretrained_model_path) pretrain_log = torch.load(opt.pretrained_model_path, map_location=opt.device)