def main():
extractor = resnet50(pretrained=True).to(DEVICE)
recurrent = utils.loadModel(
opt.model,
LSTM_Net(2048,
opt.hidden_dim,
opt.output_dim,
num_layers=opt.layers,
bias=True,
dropout=opt.dropout,
bidirectional=opt.bidirectional,
seq_predict=False)).to(DEVICE)
predict_set = dataset.TrimmedVideos(opt.video,
opt.label,
None,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]))
print("Dataset: {}".format(len(predict_set)))
predict_loader = DataLoader(predict_set,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.threads)
# Predict
predict(extractor, recurrent, predict_loader)
def main():
if not os.path.exists(opt.output):
os.makedirs(opt.output, exist_ok=True)
opt.output = os.path.join(opt.output, 'p1_valid.txt')
extractor = resnet50(pretrained=True)
classifier = utils.loadModel(opt.resume, Classifier(8192, [2048], 11))
extractor, classifier = extractor.to(DEVICE), classifier.to(DEVICE)
predict_set = dataset.TrimmedVideos(opt.video,
opt.label,
None,
sample=4,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]))
print("Dataset: {}".format(len(predict_set)))
predict_loader = DataLoader(predict_set,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.threads)
# Predict
results = predict(extractor, classifier, predict_loader)
np.savetxt(opt.output, results, fmt='%d')
print("Output File have been written to {}".format(opt.output))
def video_to_features(data_path):
""" Transfer the training set and validation set videos into features """
for train in (True, False):
datasets = dataset.TrimmedVideos(data_path,
train=train,
downsample=1,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]))
dataloader = DataLoader(datasets,
batch_size=1,
shuffle=False,
num_workers=0)
extractor = resnet50(pretrained=True).to(device).eval()
if train: train_val = "train"
else: train_val = "valid"
for index, (data, _, category, name) in enumerate(dataloader, 1):
data = data.squeeze(0)
datas = np.zeros((data.shape[0], 2048), dtype=np.float)
remain = data.shape[0]
finish = 0
while remain > 0:
step = min(remain, 50)
todo = data[finish:finish + step].to(device)
datas[finish:finish +
step] = extractor(todo).cpu().data.numpy()
remain -= step
finish += step
print("{:4d} {:16d} {}".format(
index, datas.shape,
os.path.join(data_path, "feature", train_val, category[0],
name[0] + ".npy")))
# ------------------------------------
# Save the feature tensor in .npy file
# ------------------------------------
if not os.path.exists(
os.path.join(data_path, "feature", train_val,
category[0])):
os.makedirs(
os.path.join(data_path, "feature", train_val, category[0]))
np.savetxt(os.path.join(data_path, "feature", train_val,
category[0], name[0] + ".npy"),
datas,
delimiter=',')
return
def main():
opt.output = os.path.join(opt.output, 'p2_result.txt')
extractor = resnet50(pretrained=True).to(DEVICE)
recurrent = utils.loadModel(
opt.resume,
LSTM_Net(2048,
opt.hidden_dim,
opt.output_dim,
num_layers=opt.layers,
bias=True,
dropout=opt.dropout,
bidirectional=opt.bidirectional,
seq_predict=False)).to(DEVICE)
predict_set = dataset.TrimmedVideos(opt.video,
opt.label,
None,
downsample=opt.downsample,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]))
print("Dataset: {}".format(len(predict_set)))
predict_loader = DataLoader(predict_set,
batch_size=opt.batch_size,
shuffle=False,
collate_fn=utils.collate_fn_valid,
num_workers=opt.threads)
# Predict
results = predict(extractor, recurrent, predict_loader)
np.savetxt(opt.output, results, fmt='%d')
print("Output File have been written to {}".format(opt.output))
def continuous_frame_recognition():
""" Using RNN network to recognize the action. """
start_epoch = 1
# -----------------------------------------------------
# Create Model, optimizer, scheduler, and loss function
# -----------------------------------------------------
# extractor = resnet50(pretrained=True).to(DEVICE)
recurrent = LSTM_Net(2048,
opt.hidden_dim,
opt.output_dim,
num_layers=opt.layers,
bias=True,
batch_first=False,
dropout=opt.dropout,
bidirectional=opt.bidirection,
seq_predict=False).to(DEVICE)
# ----------------------------------------------
# For signal direction LSTM
# weight_ih_l0 torch.Size([512, 2048])
# weight_hh_l0 torch.Size([512, 128])
# bias_ih_l0 torch.Size([512])
# bias_hh_l0 torch.Size([512])
#
# For bidirectional LSTM, reverse layer is added.
# weight_ih_l0_reverse torch.Size([512, 2048])
# weight_hh_l0_reverse torch.Size([512, 128])
# bias_ih_l0_reverse torch.Size([512])
# bias_hh_l0_reverse torch.Size([512])
# ----------------------------------------------
# Weight_init
if "orthogonal" in opt.weight_init:
for layer, param in recurrent.recurrent.named_parameters():
print("{} {}".format(layer, param.shape))
if len(param.shape) >= 2:
nn.init.orthogonal_(param)
# Bias_init
if "forget_bias_0" in opt.bias_init:
for layer, param in recurrent.recurrent.named_parameters():
if layer.startswith("bias"):
size = param.shape[0]
start = int(size * 0.25)
end = int(size * 0.5)
param[start:end].data.fill_(0)
if "forget_bias_1" in opt.bias_init:
for layer, param in recurrent.recurrent.named_parameters():
if layer.startswith("bias"):
size = param.shape[0]
start = int(size * 0.25)
end = int(size * 0.5)
param[start:end].data.fill_(1)
# Set optimizer
if opt.optimizer == "Adam":
optimizer = optim.Adam(recurrent.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
weight_decay=opt.weight_decay)
elif opt.optimizer == "SGD":
optimizer = optim.SGD(recurrent.parameters(),
lr=opt.lr,
momentum=opt.momentum,
weight_decay=opt.weight_decay)
elif opt.optimizer == "ASGD":
optimizer = optim.ASGD(recurrent.parameters(),
lr=opt.lr,
lambd=1e-4,
alpha=0.75,
t0=1000000.0,
weight_decay=opt.weight_decay)
elif opt.optimizer == "Adadelta":
optimizer = optim.Adadelta(recurrent.parameters(),
lr=opt.lr,
rho=0.9,
eps=1e-06,
weight_decay=opt.weight_decay)
elif opt.optimizer == "Adagrad":
optimizer = optim.Adagrad(recurrent.parameters(),
lr=opt.lr,
lr_decay=0,
weight_decay=opt.weight_decay,
initial_accumulator_value=0)
elif opt.optimizer == "SparseAdam":
optimizer = optim.SparseAdam(recurrent.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
eps=1e-08)
elif opt.optimizer == "Adamax":
optimizer = optim.Adamax(recurrent.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
eps=1e-08,
weight_decay=opt.weight_dacay)
else:
raise argparse.ArgumentError
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=opt.milestones,
gamma=opt.gamma)
# Load parameter
if opt.pretrain:
recurrent = utils.loadModel(opt.pretrain, recurrent)
if opt.resume:
recurrent, optimizer, start_epoch, scheduler = utils.loadCheckpoint(
opt.resume, recurrent, optimizer, scheduler)
# Set criterion
criterion = nn.CrossEntropyLoss().to(DEVICE)
# Set dataloader
transform = transforms.ToTensor()
trainlabel = os.path.join(opt.train, "label", "gt_train.csv")
trainfeature = os.path.join(opt.train, "feature", "train")
vallabel = os.path.join(opt.val, "label", "gt_valid.csv")
valfeature = os.path.join(opt.val, "feature", "valid")
train_set = dataset.TrimmedVideos(None,
trainlabel,
trainfeature,
downsample=opt.downsample,
transform=transform)
train_loader = DataLoader(train_set,
batch_size=opt.batch_size,
shuffle=True,
collate_fn=utils.collate_fn,
num_workers=opt.threads)
# Show the memory used by neural network
print("The neural network allocated GPU with {:.1f} MB".format(
torch.cuda.memory_allocated() / 1024 / 1024))
#------------------
# Train the models
#------------------
trainloss = []
trainaccs = []
valloss = []
valaccs = []
epochs = []
for epoch in range(start_epoch, opt.epochs + 1):
scheduler.step()
# Save the train loss and train accuracy
max_trainaccs = max(trainaccs) if len(trainaccs) else 0
min_trainloss = min(trainloss) if len(trainloss) else 0
recurrent, loss, acc = train(recurrent, train_loader, optimizer, epoch,
criterion, max_trainaccs, min_trainloss)
trainloss.append(loss)
trainaccs.append(acc)
# validate the model with several downsample ratio
loss_list, acc_list, label_list = [], [], []
for downsample in [1, 2, 4, 6, 12]:
val_set = dataset.TrimmedVideos(None,
vallabel,
valfeature,
downsample=downsample,
transform=transform)
val_loader = DataLoader(val_set,
batch_size=1,
shuffle=True,
collate_fn=utils.collate_fn,
num_workers=opt.threads)
print("[Epoch {}] [Validation] [Downsample: {:2d}]".format(
epoch, downsample))
acc, loss = val(recurrent, val_loader, epoch, criterion)
loss_list.append(loss)
acc_list.append(acc)
label_list.append('val_{}'.format(downsample))
valloss.append(loss_list)
valaccs.append(acc_list)
# Save the epochs
epochs.append(epoch)
# with open(os.path.join(opt.log, "problem_2", opt.tag, 'statistics.txt'), 'w') as textfile:
# textfile.write("\n".join(map(lambda x: str(x), (trainloss, trainaccs, valloss, valaccs, epochs))))
records = list(
map(lambda x: np.array(x),
(trainloss, trainaccs, valloss, valaccs, epochs)))
for record, name in zip(records,
('trainloss.txt', 'trainaccs.txt',
'valloss.txt', 'valaccs.txt', 'epochs.txt')):
np.savetxt(os.path.join(opt.log, "problem_2", opt.tag, name),
record)
if epoch % opt.save_interval == 0:
savepath = os.path.join(opt.checkpoints, "problem_2", opt.tag,
str(epoch) + '.pth')
utils.saveCheckpoint(savepath, recurrent, optimizer, scheduler,
epoch)
# Draw the accuracy / loss curve
draw_graphs(trainloss, valloss, trainaccs, valaccs, epochs,
"problem_2", label_list)
return recurrent
def single_frame_recognition():
""" Using 2D CNN network to recognize the action. """
#-----------------------------------------------------
# Create Model, optimizer, scheduler, and loss function
#------------------------------------------------------
extractor = resnet50(pretrained=True).to(DEVICE)
classifier = Classifier(2048 * opt.sample, [2048],
num_class=opt.output_dim).to(DEVICE)
print(classifier)
# Set optimizer
if opt.optimizer == "Adam":
optimizer = optim.Adam(classifier.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
weight_decay=opt.weight_decay)
elif opt.optimizer == "SGD":
optimizer = optim.SGD(classifier.parameters(),
lr=opt.lr,
momentum=opt.momentum,
weight_decay=opt.weight_decay)
elif opt.optimizer == "ASGD":
optimizer = optim.ASGD(classifier.parameters(),
lr=opt.lr,
lambd=1e-4,
alpha=0.75,
t0=1000000.0,
weight_decay=opt.weight_decay)
elif opt.optimizer == "Adadelta":
optimizer = optim.Adadelta(classifier.parameters(),
lr=opt.lr,
rho=0.9,
eps=1e-06,
weight_decay=opt.weight_decay)
elif opt.optimizer == "Adagrad":
optimizer = optim.Adagrad(classifier.parameters(),
lr=opt.lr,
lr_decay=0,
weight_decay=opt.weight_decay,
initial_accumulator_value=0)
elif opt.optimizer == "SparseAdam":
optimizer = optim.SparseAdam(classifier.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
eps=1e-08)
elif opt.optimizer == "Adamax":
optimizer = optim.Adamax(classifier.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
eps=1e-08,
weight_decay=opt.weight_dacay)
else:
raise argparse.ArgumentError
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=opt.milestones,
gamma=opt.gamma)
criterion = nn.CrossEntropyLoss().to(DEVICE)
transform = transforms.ToTensor()
trainlabel = os.path.join(opt.train, "label", "gt_train.csv")
trainfeature = os.path.join(opt.train, "feature", "train")
vallabel = os.path.join(opt.val, "label", "gt_valid.csv")
valfeature = os.path.join(opt.val, "feature", "valid")
train_set = dataset.TrimmedVideos(None,
trainlabel,
trainfeature,
sample=4,
transform=transform)
val_set = dataset.TrimmedVideos(None,
vallabel,
valfeature,
sample=4,
transform=transform)
train_loader = DataLoader(train_set,
batch_size=opt.batch_size,
shuffle=True,
drop_last=True,
num_workers=opt.threads)
val_loader = DataLoader(val_set,
batch_size=opt.batch_size,
shuffle=False,
drop_last=True,
num_workers=opt.threads)
# Show the memory used by neural network
print("The neural network allocated GPU with {:.1f} MB".format(
torch.cuda.memory_allocated() / 1024 / 1024))
#------------------
# Train the models
#------------------
trainloss = []
trainaccs = []
valloss = []
valaccs = []
epochs = []
for epoch in range(1, opt.epochs + 1):
scheduler.step()
# Save the train loss and train accuracy
extractor, classifier, loss, acc = train(extractor, classifier,
train_loader, optimizer,
epoch, criterion)
trainloss.append(loss)
trainaccs.append(acc)
# Save the validation loss and validation accuracy
acc, loss = val(extractor, classifier, val_loader, epoch, criterion)
valloss.append(loss)
valaccs.append(acc)
# Save the epochs
epochs.append(epoch)
records = list(
map(lambda x: np.array(x),
(trainloss, trainaccs, valloss, valaccs, epochs)))
for record, name in zip(records,
('trainloss.txt', 'trainaccs.txt',
'valloss.txt', 'valaccs.txt', 'epochs.txt')):
np.savetxt(os.path.join(opt.log, "problem_1", opt.tag, name),
record)
if epoch % opt.save_interval == 0:
savepath = os.path.join(opt.checkpoints, "problem_1", opt.tag,
str(epoch) + '.pth')
utils.saveCheckpoint(savepath, classifier, optimizer, scheduler,
epoch)
draw_graphs(trainloss, valloss, trainaccs, valaccs, epochs)
return extractor, classifier