def main():
extractor = resnet50(pretrained=True).to(DEVICE)
recurrent = utils.loadModel(
opt.model,
LSTM_Net(2048,
opt.hidden_dim,
opt.output_dim,
num_layers=opt.layers,
bias=True,
dropout=opt.dropout,
bidirectional=opt.bidirectional,
seq_predict=False)).to(DEVICE)
predict_set = dataset.TrimmedVideos(opt.video,
opt.label,
None,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]))
print("Dataset: {}".format(len(predict_set)))
predict_loader = DataLoader(predict_set,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.threads)
# Predict
predict(extractor, recurrent, predict_loader)
def main():
valids_p1 = TrimmedVideos(None, opt.label, opt.feature, sample=4, transform=transforms.ToTensor())
loader_p1 = DataLoader(valids_p1, batch_size=opt.plot_num, shuffle=False)
valids_p2 = TrimmedVideos(None, opt.label, opt.feature, downsample=12, transform=transforms.ToTensor())
loader_p2 = DataLoader(valids_p2, batch_size=1, shuffle=False)
recurrent = utils.loadModel(opt.resume,
LSTM_Net(2048, 128, 11, num_layers=2, bias=True,
dropout=0.2, bidirectional=False, seq_predict=False)
).to(DEVICE)
graph_1 = os.path.join(opt.graph, 'p1_tsne.png')
graph_2 = os.path.join(opt.graph, 'p2_tsne.png')
dimension_reduction_cnn(graph_1, loader_p1)
dimension_reduction_rnn(graph_2, loader_p2, recurrent)
def main():
opt.output = os.path.join(opt.output, 'p2_result.txt')
extractor = resnet50(pretrained=True).to(DEVICE)
recurrent = utils.loadModel(
opt.resume,
LSTM_Net(2048,
opt.hidden_dim,
opt.output_dim,
num_layers=opt.layers,
bias=True,
dropout=opt.dropout,
bidirectional=opt.bidirectional,
seq_predict=False)).to(DEVICE)
predict_set = dataset.TrimmedVideos(opt.video,
opt.label,
None,
downsample=opt.downsample,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]))
print("Dataset: {}".format(len(predict_set)))
predict_loader = DataLoader(predict_set,
batch_size=opt.batch_size,
shuffle=False,
collate_fn=utils.collate_fn_valid,
num_workers=opt.threads)
# Predict
results = predict(extractor, recurrent, predict_loader)
np.savetxt(opt.output, results, fmt='%d')
print("Output File have been written to {}".format(opt.output))
def continuous_frame_recognition():
""" Using RNN network to recognize the action. """
start_epoch = 1
# -----------------------------------------------------
# Create Model, optimizer, scheduler, and loss function
# -----------------------------------------------------
# extractor = resnet50(pretrained=True).to(DEVICE)
recurrent = LSTM_Net(2048,
opt.hidden_dim,
opt.output_dim,
num_layers=opt.layers,
bias=True,
batch_first=False,
dropout=opt.dropout,
bidirectional=opt.bidirection,
seq_predict=False).to(DEVICE)
# ----------------------------------------------
# For signal direction LSTM
# weight_ih_l0 torch.Size([512, 2048])
# weight_hh_l0 torch.Size([512, 128])
# bias_ih_l0 torch.Size([512])
# bias_hh_l0 torch.Size([512])
#
# For bidirectional LSTM, reverse layer is added.
# weight_ih_l0_reverse torch.Size([512, 2048])
# weight_hh_l0_reverse torch.Size([512, 128])
# bias_ih_l0_reverse torch.Size([512])
# bias_hh_l0_reverse torch.Size([512])
# ----------------------------------------------
# Weight_init
if "orthogonal" in opt.weight_init:
for layer, param in recurrent.recurrent.named_parameters():
print("{} {}".format(layer, param.shape))
if len(param.shape) >= 2:
nn.init.orthogonal_(param)
# Bias_init
if "forget_bias_0" in opt.bias_init:
for layer, param in recurrent.recurrent.named_parameters():
if layer.startswith("bias"):
size = param.shape[0]
start = int(size * 0.25)
end = int(size * 0.5)
param[start:end].data.fill_(0)
if "forget_bias_1" in opt.bias_init:
for layer, param in recurrent.recurrent.named_parameters():
if layer.startswith("bias"):
size = param.shape[0]
start = int(size * 0.25)
end = int(size * 0.5)
param[start:end].data.fill_(1)
# Set optimizer
if opt.optimizer == "Adam":
optimizer = optim.Adam(recurrent.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
weight_decay=opt.weight_decay)
elif opt.optimizer == "SGD":
optimizer = optim.SGD(recurrent.parameters(),
lr=opt.lr,
momentum=opt.momentum,
weight_decay=opt.weight_decay)
elif opt.optimizer == "ASGD":
optimizer = optim.ASGD(recurrent.parameters(),
lr=opt.lr,
lambd=1e-4,
alpha=0.75,
t0=1000000.0,
weight_decay=opt.weight_decay)
elif opt.optimizer == "Adadelta":
optimizer = optim.Adadelta(recurrent.parameters(),
lr=opt.lr,
rho=0.9,
eps=1e-06,
weight_decay=opt.weight_decay)
elif opt.optimizer == "Adagrad":
optimizer = optim.Adagrad(recurrent.parameters(),
lr=opt.lr,
lr_decay=0,
weight_decay=opt.weight_decay,
initial_accumulator_value=0)
elif opt.optimizer == "SparseAdam":
optimizer = optim.SparseAdam(recurrent.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
eps=1e-08)
elif opt.optimizer == "Adamax":
optimizer = optim.Adamax(recurrent.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
eps=1e-08,
weight_decay=opt.weight_dacay)
else:
raise argparse.ArgumentError
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=opt.milestones,
gamma=opt.gamma)
# Load parameter
if opt.pretrain:
recurrent = utils.loadModel(opt.pretrain, recurrent)
if opt.resume:
recurrent, optimizer, start_epoch, scheduler = utils.loadCheckpoint(
opt.resume, recurrent, optimizer, scheduler)
# Set criterion
criterion = nn.CrossEntropyLoss().to(DEVICE)
# Set dataloader
transform = transforms.ToTensor()
trainlabel = os.path.join(opt.train, "label", "gt_train.csv")
trainfeature = os.path.join(opt.train, "feature", "train")
vallabel = os.path.join(opt.val, "label", "gt_valid.csv")
valfeature = os.path.join(opt.val, "feature", "valid")
train_set = dataset.TrimmedVideos(None,
trainlabel,
trainfeature,
downsample=opt.downsample,
transform=transform)
train_loader = DataLoader(train_set,
batch_size=opt.batch_size,
shuffle=True,
collate_fn=utils.collate_fn,
num_workers=opt.threads)
# Show the memory used by neural network
print("The neural network allocated GPU with {:.1f} MB".format(
torch.cuda.memory_allocated() / 1024 / 1024))
#------------------
# Train the models
#------------------
trainloss = []
trainaccs = []
valloss = []
valaccs = []
epochs = []
for epoch in range(start_epoch, opt.epochs + 1):
scheduler.step()
# Save the train loss and train accuracy
max_trainaccs = max(trainaccs) if len(trainaccs) else 0
min_trainloss = min(trainloss) if len(trainloss) else 0
recurrent, loss, acc = train(recurrent, train_loader, optimizer, epoch,
criterion, max_trainaccs, min_trainloss)
trainloss.append(loss)
trainaccs.append(acc)
# validate the model with several downsample ratio
loss_list, acc_list, label_list = [], [], []
for downsample in [1, 2, 4, 6, 12]:
val_set = dataset.TrimmedVideos(None,
vallabel,
valfeature,
downsample=downsample,
transform=transform)
val_loader = DataLoader(val_set,
batch_size=1,
shuffle=True,
collate_fn=utils.collate_fn,
num_workers=opt.threads)
print("[Epoch {}] [Validation] [Downsample: {:2d}]".format(
epoch, downsample))
acc, loss = val(recurrent, val_loader, epoch, criterion)
loss_list.append(loss)
acc_list.append(acc)
label_list.append('val_{}'.format(downsample))
valloss.append(loss_list)
valaccs.append(acc_list)
# Save the epochs
epochs.append(epoch)
# with open(os.path.join(opt.log, "problem_2", opt.tag, 'statistics.txt'), 'w') as textfile:
# textfile.write("\n".join(map(lambda x: str(x), (trainloss, trainaccs, valloss, valaccs, epochs))))
records = list(
map(lambda x: np.array(x),
(trainloss, trainaccs, valloss, valaccs, epochs)))
for record, name in zip(records,
('trainloss.txt', 'trainaccs.txt',
'valloss.txt', 'valaccs.txt', 'epochs.txt')):
np.savetxt(os.path.join(opt.log, "problem_2", opt.tag, name),
record)
if epoch % opt.save_interval == 0:
savepath = os.path.join(opt.checkpoints, "problem_2", opt.tag,
str(epoch) + '.pth')
utils.saveCheckpoint(savepath, recurrent, optimizer, scheduler,
epoch)
# Draw the accuracy / loss curve
draw_graphs(trainloss, valloss, trainaccs, valaccs, epochs,
"problem_2", label_list)
return recurrent
print("loading data ...") # 把'training_label.txt'跟'training_nolabel.txt'讀進來
train_x, y = load_training_data(train_with_label)
#train_x_no_label = load_training_data(train_no_label)
#train_x_no_label = train_x_no_label[:160000]
# 對input跟labels做預處理
#preprocess = Preprocess(train_x, sen_len, train_x_no_label, w2v_path=w2v_path)
preprocess = Preprocess(train_x, sen_len, w2v_path=w2v_path)
embedding = preprocess.make_embedding(load=True)
#print(preprocess.embedding.most_similar("love"))
train_x = preprocess.sentence_word2idx()
#train_x, train_x_no_label = preprocess.sentence_word2idx()
y = preprocess.labels_to_tensor(y)
# 製作一個model的對象3model = LSTM_Net(embedding, embedding_dim=250, hidden_dim=150, num_layers=1, dropout=0.1, fix_embedding=fix_embedding)
model = LSTM_Net(embedding,
embedding_dim=256,
hidden_dim=128,
num_layers=3,
dropout=0.2,
fix_embedding=fix_embedding)
model = model.to(device)
# 把data分為training data跟validation data(將一部份training data拿去當作validation data)
#X_train, X_val, y_train, y_val = train_x[40000:], train_x[:40000], y[40000:], y[:40000]
X_train, X_val, y_train, y_val = train_x[:160000], train_x[
160000:], y[:160000], y[160000:]
# 把data做成dataset供dataloader取用
#train_dataset = TwitterDataset(X=X_train, y=y_train, unlabel_x = train_x_no_label)
train_dataset = TwitterDataset(X=X_train, y=y_train)
val_dataset = TwitterDataset(X=X_val, y=y_val)
#train_no_label_dataset = TwitterDataset(X=train_no_label_x, y=None)
def temporal_action_segmentation():
""" Using RNN network to segmentation the action. """
start_epoch = 1
#------------------------------------------------------
# Create Model, optimizer, scheduler, and loss function
#------------------------------------------------------
recurrent = LSTM_Net(2048,
opt.hidden_dim,
opt.output_dim,
num_layers=opt.layers,
bias=True,
batch_first=False,
dropout=opt.dropout,
bidirectional=opt.bidirection,
seq_predict=True).to(DEVICE)
# Weight_init
if "orthogonal" in opt.weight_init:
for layer, param in recurrent.recurrent.named_parameters():
print("{} {}".format(layer, param.shape))
if len(param.shape) >= 2:
nn.init.orthogonal_(param)
# Bias_init
if "forget_bias_0" in opt.bias_init:
for layer, param in recurrent.recurrent.named_parameters():
if layer.startswith("bias"):
start = int(param.shape[0] * 0.25)
end = int(param.shape[0] * 0.5)
param[start:end].data.fill_(0)
if "forget_bias_1" in opt.bias_init:
for layer, param in recurrent.recurrent.named_parameters():
if layer.startswith("bias"):
start = int(param.shape[0] * 0.25)
end = int(param.shape[0] * 0.5)
param[start:end].data.fill_(1)
# Set optimizer
if opt.optimizer == "Adam":
optimizer = optim.Adam(recurrent.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
weight_decay=opt.weight_decay)
elif opt.optimizer == "SGD":
optimizer = optim.SGD(recurrent.parameters(),
lr=opt.lr,
momentum=opt.momentum,
weight_decay=opt.weight_decay)
elif opt.optimizer == "ASGD":
optimizer = optim.ASGD(recurrent.parameters(),
lr=opt.lr,
lambd=1e-4,
alpha=0.75,
t0=1000000.0,
weight_decay=opt.weight_decay)
elif opt.optimizer == "Adadelta":
optimizer = optim.Adadelta(recurrent.parameters(),
lr=opt.lr,
rho=0.9,
eps=1e-06,
weight_decay=opt.weight_decay)
elif opt.optimizer == "Adagrad":
optimizer = optim.Adagrad(recurrent.parameters(),
lr=opt.lr,
lr_decay=0,
weight_decay=opt.weight_decay,
initial_accumulator_value=0)
elif opt.optimizer == "SparseAdam":
optimizer = optim.SparseAdam(recurrent.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
eps=1e-08)
elif opt.optimizer == "Adamax":
optimizer = optim.Adamax(recurrent.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2),
eps=1e-08,
weight_decay=opt.weight_dacay)
else:
raise argparse.ArgumentError
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=opt.milestones,
gamma=opt.gamma)
# Load parameter
if opt.pretrain:
recurrent = utils.loadModel(opt.pretrain, recurrent)
print("Loaded pretrain model: {}".format(opt.pretrain))
if opt.resume:
recurrent, optimizer, start_epoch, scheduler = utils.loadCheckpoint(
opt.resume, recurrent, optimizer, scheduler)
print("Resume training: {}".format(opt.resume))
# Set criterion
criterion = nn.CrossEntropyLoss().to(DEVICE)
# Set dataloader
transform = transforms.ToTensor()
trainlabel = os.path.join(opt.train, "labels", "train")
trainfeature = os.path.join(opt.train, "feature", "train")
vallabel = os.path.join(opt.val, "labels", "valid")
valfeature = os.path.join(opt.val, "feature", "valid")
train_set = dataset.FullLengthVideos(
None,
trainlabel,
trainfeature,
downsample=opt.train_downsample,
transform=transform,
summarize=opt.summarize,
sampling=opt.sampling,
)
train_loader = DataLoader(train_set,
batch_size=opt.batch_size,
shuffle=True,
collate_fn=utils.collate_fn_seq,
num_workers=opt.threads)
val_set = dataset.FullLengthVideos(
None,
vallabel,
valfeature,
downsample=opt.val_downsample,
transform=transform,
summarize=None,
sampling=0,
)
val_loader = DataLoader(val_set,
batch_size=1,
shuffle=False,
collate_fn=utils.collate_fn_seq,
num_workers=opt.threads)
val_set_2 = dataset.FullLengthVideos(None,
vallabel,
valfeature,
downsample=opt.train_downsample,
transform=transform,
summarize=None,
sampling=0)
val_loader_2 = DataLoader(val_set_2,
batch_size=1,
shuffle=False,
collate_fn=utils.collate_fn_seq,
num_workers=opt.threads)
# Show the memory used by neural network
print("The neural network allocated GPU with {:.1f} MB".format(
torch.cuda.memory_allocated() / 1024 / 1024))
#------------------
# Train the models
#------------------
trainloss, trainaccs, valloss, valaccs = [], [], [], []
epochs = []
categories = [name.split('.')[0] for name in os.listdir(valfeature)]
# Pre-test of the pretrain model
acc, loss = val(recurrent, val_loader, 0, criterion)
valloss.append(loss)
valaccs.append(acc)
epochs.append(0)
for epoch in range(start_epoch, opt.epochs + 1):
scheduler.step()
# Save the train loss and train accuracy
max_trainaccs = max(trainaccs) if len(trainaccs) > 0 else 0
min_trainloss = min(trainloss) if len(trainloss) > 0 else 0
recurrent, acc, loss = train(recurrent, train_loader, optimizer, epoch,
criterion, max_trainaccs, min_trainloss)
trainloss.append(loss)
trainaccs.append(acc)
# validate the model with several downsample ratio
acc, loss = val(recurrent, val_loader, epoch, criterion)
valloss.append(loss)
valaccs.append(acc)
acc, loss = val(recurrent,
val_loader_2,
epoch,
criterion,
visual=False)
# Save the epochs
epochs.append(epoch)
for x, y in ((trainloss, "trainloss.txt"),
(trainaccs, "trainaccs.txt"), (valloss, "valloss.txt"),
(valaccs, "valaccs.txt"), (epochs, "epochs.txt")):
np.savetxt(os.path.join(opt.log, "problem_3", opt.tag, y),
np.array(x))
if epoch % opt.save_interval == 0:
savepath = os.path.join(opt.checkpoints, "problem_3", opt.tag,
str(epoch) + '.pth')
utils.saveCheckpoint(savepath, recurrent, optimizer, scheduler,
epoch)
# Draw the accuracy / loss curve
draw_graphs(trainloss,
valloss,
trainaccs,
valaccs,
epochs,
label=categories)
return recurrent
test_x = load_testing_data(testing_data)
preprocess = Preprocess(test_x, sen_len, w2v_path=w2v_path)
embedding = preprocess.make_embedding(load=True)
test_x = preprocess.sentence_word2idx()
test_dataset = TwitterDataset(X=test_x, y=None)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=8,
collate_fn=collate_fn)
print('\nload LSTM Model ...')
#model = torch.load(os.path.join(model_dir, 'ckpt.model'))
model = LSTM_Net(embedding,
embedding_dim=256,
hidden_dim=128,
num_layers=3,
dropout=0.2,
fix_embedding=True)
model = model.to(
device) # device為"cuda"，model使用GPU來訓練(餵進去的inputs也需要是cuda tensor)
model.load_state_dict(
torch.load(os.path.join(model_dir, 'ckpt_1.model'),
map_location={'cuda:3': 'cuda'}))
output1 = testing(batch_size, test_loader, model, device, step=1)
print('load BI-LSTM-1 Model ...')
#model = torch.load(os.path.join(model_dir, 'ckpt.model'))
model = BILSTM_Net(embedding,
embedding_dim=256,
hidden_dim=128,
num_layers=3,