def test(dataset="UCF-101"):
dataset = dataset.lower()
with tf.Graph().as_default():
images_placeholder, labels_placeholder = placeholder_inputs()
c3d = C3D.C3D(num_class=101)
logit = c3d.model(images_placeholder, batch_size=BATCH_SIZE)
correct_pred = tf.reduce_sum(
tf.cast(tf.equal(tf.argmax(logit, 1), labels_placeholder),
tf.int16))
print(correct_pred)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, model_save_dir + '/' + dataset + '.model')
test_data = dataReader.dataReader("data/" + dataset + "_test_list.txt",
batch_size=BATCH_SIZE)
finished = False
cnt = 0
total = 0
while (not finished):
test_images, test_labels = test_data.get_next_batch()
finished = test_data.is_finished()
_correct_pred = sess.run([correct_pred],
feed_dict={
images_placeholder: test_images,
labels_placeholder: test_labels
})
total += test_images.shape[0]
cnt += _correct_pred[0]
print(cnt, total, cnt / float(total))
def main(args):
# Create model directory for saving trained models
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
transform = None
# Build data loader
data_loader = get_loader(root=args.video_root,
list_file=args.list_file,
transform=transform,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
# Build the models
model = C3D(args.num_classes).to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
params = model.parameters()
optimizer = torch.optim.Adam(params, lr=args.learning_rate)
# Train the models
total_step = len(data_loader)
for epoch in range(args.num_epochs):
for i, sample in enumerate(data_loader):
# Set mini-batch dataset
clip, target = sample
clip = clip.to(device)
target = target.to(device)
# clip = sample['clip'].to(device)
# target = sample['label'].to(device)
# Forward, backward and optimize
outputs = model(clip)
# preds = torch.max(outputs, 1)[1]
loss = criterion(outputs, target)
# zero the parameter gradients
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Print log info
if i % args.log_step == 0:
print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(
epoch, args.num_epochs, i, total_step, loss.item()))
# Save the model checkpoints
if (i + 1) % args.save_step == 0:
torch.save(
model.state_dict(),
os.path.join(args.model_path,
'c3d-{}-{}.ckpt'.format(epoch + 1, i + 1)))
def predict():
X = get_sport_clip('test')
X = Variable(X)
X = X.cuda()
# get network pretrained model
net = C3D()
net.load_state_dict(torch.load('c3d.pickle'))
net.cuda()
net.eval()
# perform prediction
prediction = net(X)
prediction = prediction.data.cpu().numpy()
# read labels
labels = read_labels_from_file('labels.txt')
# print top predictions
top_inds = prediction[0].argsort()[::-1][:5] # reverse sort and take five largest items
print('\nTop 5:')
for i in top_inds:
print('{:.5f} {}'.format(prediction[0][i], labels[i]))
def init_model():
# init model
return C3D(num_classes=101, pretrained=True)
def main(config):
if config.model == 'c3d':
model, params = C3D(config)
elif config.model == 'convlstm':
model, params = ConvLSTM(config)
elif config.model == 'densenet':
model, params = densenet(config)
elif config.model == 'densenet_lean':
model, params = densenet_lean(config)
elif config.model == 'resnext':
model, params = resnext(config)
else:
model, params = densenet_lean(config)
dataset = config.dataset
sample_size = config.sample_size
stride = config.stride
sample_duration = config.sample_duration
cv = config.num_cv
# crop_method = GroupRandomScaleCenterCrop(size=sample_size)
crop_method = MultiScaleRandomCrop(config.scales, config.sample_size[0])
# norm = Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
norm = Normalize([114.7748, 107.7354, 99.475], [1, 1, 1])
# spatial_transform = Compose(
# [crop_method,
# GroupRandomHorizontalFlip(),
# ToTensor(1), norm])
spatial_transform = Compose([
RandomHorizontalFlip(), crop_method,
ToTensor(config.norm_value), norm
])
# temporal_transform = RandomCrop(size=sample_duration, stride=stride)
temporal_transform = TemporalRandomCrop(config.sample_duration,
config.downsample)
target_transform = Label()
train_batch = config.train_batch
train_data = RWF2000('/content/RWF_2000/frames/',
g_path + '/RWF-2000.json', 'training',
spatial_transform, temporal_transform,
target_transform, dataset)
train_loader = DataLoader(train_data,
batch_size=train_batch,
shuffle=True,
num_workers=4,
pin_memory=True)
crop_method = GroupScaleCenterCrop(size=sample_size)
norm = Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
spatial_transform = Compose([crop_method, ToTensor(), norm])
temporal_transform = CenterCrop(size=sample_duration, stride=stride)
target_transform = Label()
val_batch = config.val_batch
val_data = RWF2000('/content/RWF_2000/frames/', g_path + '/RWF-2000.json',
'validation', spatial_transform, temporal_transform,
target_transform, dataset)
val_loader = DataLoader(val_data,
batch_size=val_batch,
shuffle=False,
num_workers=4,
pin_memory=True)
if not os.path.exists('{}/pth'.format(config.output)):
os.mkdir('{}/pth'.format(config.output))
if not os.path.exists('{}/log'.format(config.output)):
os.mkdir('{}/log'.format(config.output))
batch_log = Log(
'{}/log/{}_fps{}_{}_batch{}.log'.format(
config.output,
config.model,
sample_duration,
dataset,
cv,
), ['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
epoch_log = Log(
'{}/log/{}_fps{}_{}_epoch{}.log'.format(config.output, config.model,
sample_duration, dataset, cv),
['epoch', 'loss', 'acc', 'lr'])
val_log = Log(
'{}/log/{}_fps{}_{}_val{}.log'.format(config.output, config.model,
sample_duration, dataset, cv),
['epoch', 'loss', 'acc'])
criterion = nn.CrossEntropyLoss().to(device)
# criterion = nn.BCELoss().to(device)
learning_rate = config.learning_rate
momentum = config.momentum
weight_decay = config.weight_decay
optimizer = torch.optim.SGD(params=params,
lr=learning_rate,
momentum=momentum,
weight_decay=weight_decay,
dampening=False,
nesterov=False)
# optimizer = torch.optim.Adam(params=params, lr = learning_rate, weight_decay= weight_decay)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, verbose=True, factor=config.factor, min_lr=config.min_lr)
acc_baseline = config.acc_baseline
loss_baseline = 1
for p in range(1, config.num_prune):
if p > 0:
model = torch.load('{}/pth/prune_{}.pth'.format(
config.output, p - 1))
print(f"Prune {p}/{config.num_prune}")
params = sum([np.prod(p.size()) for p in model.parameters()])
print("Number of Parameters: %.1fM" % (params / 1e6))
model = prune_model(model)
params = sum([np.prod(p.size()) for p in model.parameters()])
print("Number of Parameters: %.1fM" % (params / 1e6))
model.to(config.device)
acc_baseline = 0
for i in range(5):
train(i, train_loader, model, criterion, optimizer, device,
batch_log, epoch_log)
val_loss, val_acc = val(i, val_loader, model, criterion, device,
val_log)
scheduler.step(val_loss)
if val_acc > acc_baseline or (val_acc >= acc_baseline
and val_loss < loss_baseline):
# torch.save(
# model.state_dict(),
# '{}/pth/prune_{}_{}_fps{}_{}{}_{}_{:.4f}_{:.6f}.pth'.format(
# config.output, p, config.model, sample_duration, dataset, cv, i, val_acc,
# val_loss))
torch.save(model,
'{}/pth/prune_{}.pth'.format(config.output, p))
def train(dataset="UCF-101"):
# Get the sets of images and labels for training, validation, and
# Tell TensorFlow that the model will be built into the default Graph.
dataset = dataset.lower()
# Create model directory
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
# use_pretrained_model = True
# model_filename = "./sports1m_finetuning_ucf101.model"
with tf.Graph().as_default():
images_placeholder, labels_placeholder = placeholder_inputs()
lr = tf.placeholder(tf.float32)
c3d = C3D.C3D(num_class=101)
logit = c3d.model(images_placeholder, batch_size=BATCH_SIZE)
loss, accuracy, pred = get_loss_accuracy(logit, labels_placeholder)
train_op = tf.train.AdamOptimizer(lr).minimize(loss)
train_data = dataReader.dataReader("data/" + dataset +
"_train_list.txt",
batch_size=BATCH_SIZE)
test_data = dataReader.dataReader("data/" + dataset + "_test_list.txt",
batch_size=BATCH_SIZE)
init = tf.global_variables_initializer()
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
sess.run(init)
saver = tf.train.Saver()
# ckpt=tf.train.get_checkpoint_state(model_save_dir)
# if ckpt and ckpt.model_checkpoint_path:
# print(ckpt.model_checkpoint_path)
# saver.restore(sess, ckpt.model_checkpoint_path)
# Create a session for running Ops on the Graph.
learning_rate = 0.003
for step in range(FLAGS.max_steps):
if step != 0 and step % 2e+5 == 0: learning_rate /= 10
# start_time = time.time()
train_images, train_labels = train_data.get_next_batch()
sess.run(train_op,
feed_dict={
images_placeholder: train_images,
labels_placeholder: train_labels,
lr: learning_rate
})
# duration = time.time() - start_time
# print('Step %d: %.3f sec' % (step, duration))
# Save a checkpoint and evaluate the model periodically.
if (step) % 10 == 0 or (step + 1) == FLAGS.max_steps:
print('step %d Train' % (step))
acc, _loss, _pred, _logit = sess.run(
[accuracy, loss, pred, logit],
feed_dict={
images_placeholder: train_images,
labels_placeholder: train_labels,
lr: learning_rate
})
print(_logit)
# print (_pred)
print("accuracy: " + "{:.5f}".format(acc) + " loss:" +
"{:.5f}".format(_loss))
if (step) % 100 == 0 or (step + 1) == FLAGS.max_steps:
print('step %d Test' % (step))
test_images, test_labels = test_data.get_next_batch()
_loss, acc = sess.run(
[loss, accuracy],
feed_dict={
images_placeholder: test_images,
labels_placeholder: test_labels,
lr: learning_rate
})
print("accuracy: " + "{:.5f}".format(acc) + " loss:" +
"{:.5f}".format(_loss))
saver.save(sess, model_save_dir + '/' + dataset + '.model')
print("done")
# Creating PT data samplers and loaders:
train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(val_indices)
train_loader = torch.utils.data.DataLoader(trainset,
batch_size=batch_size,
sampler=train_sampler,
num_workers=8)
validation_loader = torch.utils.data.DataLoader(trainset,
batch_size=batch_size,
sampler=valid_sampler,
num_workers=8)
#creating network
net = C3D()
if args.resume:
print('==> Resuming from checkpoint..')
checkpoint = torch.load('./checkpoint/ckpt.pth')
net.load_state_dict(checkpoint['net'])
best_loss = checkpoint['loss']
start_epoch = checkpoint['epoch']
net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
net.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(),
lr=args.lr,
momentum=0.9,
def fine_tune(load_dir, save_dir, filename, dataloader_train, dataloader_train_sampler,
dataloader_val):
# load pretrained model
pretrained_weights = torch.load(os.path.join(load_dir, filename), map_location='cpu')
feature_state_dict = {key: value for key, value in pretrained_weights.items()
if key not in ['fc8.weight', 'fc8.bias']}
net = C3D.C3D()
state_dict = deepcopy(net.state_dict())
state_dict.update(feature_state_dict)
net.load_state_dict(state_dict)
net.to(device)
net.train()
# pdb.set_trace()
summary(net, (3,args.n_frames_per_clip,112,112))
# pdb.set_trace()
num_epochs = 100
step = 0
# determine optimizer
criterion = nn.CrossEntropyLoss()
fc_lr_layers = list(map(id, net.fc8.parameters()))
pretrained_lr_layers = [p for p in net.parameters()
if id(p) not in fc_lr_layers and p.requires_grad==True]
# pretrained_lr_layers = filter(lambda p:
# id(p) not in fc_lr_layers, model.parameters())
optimizer = torch.optim.SGD([
{"params": net.fc8.parameters()},
{"params": pretrained_lr_layers, "lr": 1e-4, 'weight_decay':5e-3}
], lr=1e-3, momentum=0.9, weight_decay=1e-2)
# optimizer = torch.optim.SGD(net.fc8.parameters(), lr=1e-3, momentum=0.9, weight_decay=1e-5)
# optimizer = torch.optim.Adam([
# {"params": model.module.fc.parameters()},
# {"params": pretrained_lr_layers, "lr": 1e-4}
# ], lr=1e-3)
# optimizer = optim.SGD(model.parameters(), lr=0.001,momentum=0.9, weight_decay=5e-4)
train_logger = utils.Logger(os.path.join('output', 'C3D-fine-tune-all.log'),
['step', 'train_loss', 'train_acc', 'val_loss', 'val_acc',
'lr_feature', 'lr_fc'])
# scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', factor=0.5, patience=1)
scheduler = lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.1)
train_loss = utils.AverageMeter()
train_acc = utils.AverageMeter()
val_loss = utils.AverageMeter()
val_acc = utils.AverageMeter()
batch_cache = 1
for epoch in trange(num_epochs): # loop over the dataset multiple times
train_loss.reset()
train_acc.reset()
for data in dataloader_train:
outputs, logits, labels = forward(net, data)
# print(batch_cache)
# pdb.set_trace()
# if batch_cache == 8: # accumulate number of batches because gpu memory limits the batch size
# optimizer.step()
# optimizer.zero_grad()
loss_ = criterion(logits, labels)
optimizer.zero_grad()
loss_.backward()
optimizer.step()
train_loss.update(loss_.item())
train_acc.update(utils.calculate_accuracy(outputs, labels))
# if batch_cache == 8:
# batch_cache = 0
if step % 100 == 0:
val_loss.reset()
val_acc.reset()
net.eval()
# for data_sampler in dataloader_train_sampler:
# outputs_sampler, logits_sampler, labels_sampler = forward(net, data_sampler)
# loss_sampler_ = criterion(logits_sampler, labels_sampler)
# train_loss.update(loss_sampler_.item())
# train_acc.update(utils.calculate_accuracy(outputs_sampler, labels_sampler))
for data_val in dataloader_val:
outputs_val, logits_val, labels_val = forward(net, data_val)
val_loss_ = criterion(logits_val, labels_val)
val_loss.update(val_loss_.item())
val_acc.update(utils.calculate_accuracy(outputs_val, labels_val))
net.train()
print('epoch{}/{} train_acc:{:.3f} train_loss:{:.3f} val_acc:{:.3f} val_loss:{:.3f}'.format(
epoch + 1, num_epochs,
train_acc.val, train_loss.val,
val_acc.avg, val_loss.avg
))
train_logger.log({
'step': step,
'train_loss': train_loss.val,
'train_acc': train_acc.val,
'val_loss': val_loss.avg,
'val_acc': val_acc.avg,
# 'lr_feature': optimizer.param_groups[1]['lr'],
'lr_feature': 0,
'lr_fc': optimizer.param_groups[0]['lr']
})
if step % 200 == 0:
utils.save_checkpoint(net, optimizer, step, save_dir,
'c3d.pth')
step += 1
# batch_cache += 1
scheduler.step()
def train_model(dataset=dataset,
save_dir=save_dir,
num_classes=num_classes,
lr=lr,
num_epochs=nEpochs):
"""
Args:
num_classes (int): Number of classes in the data
num_epochs (int, optional): Number of epochs to train for.
"""
model = C3D(num_classes=num_classes, pretrained=True)
train_params = [{
'params': C3D.get_1x_lr_params(model),
'lr': lr
}, {
'params': C3D.get_10x_lr_params(model),
'lr': lr * 10
}]
criterion = nn.CrossEntropyLoss(
) # standard crossentropy loss for classification
optimizer = optim.SGD(train_params, lr=lr, momentum=0.9, weight_decay=5e-4)
scheduler = optim.lr_scheduler.StepLR(
optimizer, step_size=10,
gamma=0.1) # the scheduler divides the lr by 10 every 10 epochs
# if resume_epoch == 0:
# print("Training {} from scratch...".format(modelName))
# else:
# checkpoint = torch.load(os.path.join(save_dir, 'models', saveName + '_epoch-' + str(resume_epoch - 1) + '.pth.tar'),
# map_location=lambda storage, loc: storage) # Load all tensors onto the CPU
# print("Initializing weights from: {}...".format(
# os.path.join(save_dir, 'models', saveName + '_epoch-' + str(resume_epoch - 1) + '.pth.tar')))
# model.load_state_dict(checkpoint['state_dict'])
# optimizer.load_state_dict(checkpoint['opt_dict'])
# print('Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0))
model.to(device)
criterion.to(device)
# log_dir = os.path.join(save_dir, 'models', datetime.now().strftime('%b%d_%H-%M-%S') + '_' + socket.gethostname())
print('Training model on {} dataset...'.format(dataset))
train_dataloader = DataLoader(VideoDataset(dataset=dataset,
split='train',
clip_len=16),
batch_size=20,
shuffle=True,
num_workers=4)
val_dataloader = DataLoader(VideoDataset(dataset=dataset,
split='val',
clip_len=16),
batch_size=20,
num_workers=4)
test_dataloader = DataLoader(VideoDataset(dataset=dataset,
split='test',
clip_len=16),
batch_size=20,
num_workers=4)
trainval_loaders = {'train': train_dataloader, 'val': val_dataloader}
trainval_sizes = {
x: len(trainval_loaders[x].dataset)
for x in ['train', 'val']
}
test_size = len(test_dataloader.dataset)
train_acc_list, train_loss_list, val_acc_list, val_loss_list = [], [], [], []
for epoch in range(num_epochs):
# each epoch has a training and validation step
for phase in ['train', 'val']:
# start_time = timeit.default_timer()
# reset the running loss and corrects
running_loss = 0.0
running_corrects = 0.0
# set model to train() or eval() mode depending on whether it is trained
# or being validated. Primarily affects layers such as BatchNorm or Dropout.
if phase == 'train':
# scheduler.step() is to be called once every epoch during training
scheduler.step()
model.train()
else:
model.eval()
for inputs, labels in tqdm(trainval_loaders[phase]):
# move inputs and labels to the device the training is taking place on
inputs = Variable(inputs, requires_grad=True).to(device)
labels = Variable(labels).long().to(device)
optimizer.zero_grad()
if phase == 'train':
outputs = model(inputs)
else:
with torch.no_grad():
outputs = model(inputs)
probs = nn.Softmax(dim=1)(outputs)
preds = torch.max(probs, 1)[1]
loss = criterion(outputs, labels)
if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / trainval_sizes[phase]
epoch_acc = running_corrects.double() / trainval_sizes[phase]
if phase == 'train':
train_acc_list.append(epoch_acc)
train_loss_list.append(epoch_loss)
else:
val_acc_list.append(epoch_acc)
val_loss_list.append(epoch_loss)
# if phase == 'train':
# writer.add_scalar('data/train_loss_epoch', epoch_loss, epoch)
# writer.add_scalar('data/train_acc_epoch', epoch_acc, epoch)
# else:
# writer.add_scalar('data/val_loss_epoch', epoch_loss, epoch)
# writer.add_scalar('data/val_acc_epoch', epoch_acc, epoch)
print("[{}] Epoch: {}/{} Loss: {} Acc: {}".format(
phase, epoch + 1, nEpochs, epoch_loss, epoch_acc))
if phase == 'val':
torch.save(
model, '{}model_epoch{}_accuracy_{}.pkl'.format(
path, epoch + 1, epoch_acc))
# stop_time = timeit.default_timer()
# print("Execution time: " + str(stop_time - start_time) + "\n")
#
# if epoch % save_epoch == (save_epoch - 1):
# torch.save({
# 'epoch': epoch + 1,
# 'state_dict': model.state_dict(),
# 'opt_dict': optimizer.state_dict(),
# }, os.path.join(save_dir, 'models', saveName + '_epoch-' + str(epoch) + '.pth.tar'))
# print("Save model at {}\n".format(os.path.join(save_dir, 'models', saveName + '_epoch-' + str(epoch) + '.pth.tar')))
# if useTest and epoch % test_interval == (test_interval - 1):
# model.eval()
# start_time = timeit.default_timer()
#
# running_loss = 0.0
# running_corrects = 0.0
#
# for inputs, labels in tqdm(test_dataloader):
# inputs = inputs.to(device)
# labels = labels.to(device)
#
# with torch.no_grad():
# outputs = model(inputs)
# probs = nn.Softmax(dim=1)(outputs)
# preds = torch.max(probs, 1)[1]
# loss = criterion(outputs, labels)
#
# running_loss += loss.item() * inputs.size(0)
# running_corrects += torch.sum(preds == labels.data)
#
# epoch_loss = running_loss / test_size
# epoch_acc = running_corrects.double() / test_size
#
# writer.add_scalar('data/test_loss_epoch', epoch_loss, epoch)
# writer.add_scalar('data/test_acc_epoch', epoch_acc, epoch)
#
# print("[test] Epoch: {}/{} Loss: {} Acc: {}".format(epoch+1, nEpochs, epoch_loss, epoch_acc))
# stop_time = timeit.default_timer()
# print("Execution time: " + str(stop_time - start_time) + "\n")
#
# writer.close()
return train_acc_list, train_loss_list, val_acc_list, val_loss_list
print('video_path: ', opt.video_path)
print('test_data_path: ', test_data_path)
print('annotation_test_path: ', opt.annotation_test_path)
print('meta_test_path: ', opt.meta_test_path)
print('log_path: ', opt.log_path)
print('log_test_path: ', opt.log_test_path)
print('pretrained_model_path:', opt.pretrained_model_path)
test_dataset = Dataset(test_data_path, opt.annotation_path, opt.labels_path)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=1,
shuffle=False,
num_workers=16,
pin_memory=True)
model = C3D()
device = opt.device
if opt.is_parallel:
opt.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
print("multi GPUs: ", torch.cuda.device_count())
elif device != "":
opt.device = "cuda:" + str(device)
else:
opt.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if opt.pretrained_model_path != '':
print("load from ", opt.pretrained_model_path)
pretrain_log = torch.load(opt.pretrained_model_path, map_location=opt.device)