def noise_tensor(size):
n = Variable(torch.randn(size, 150))
n.cpu()
print(str(n.size()) + " noise size")
if torch.cuda.is_available():
return n.cuda()
return n
def noise_tensor(size):
n = Variable(torch.randn(size, 100))
n.cpu()
if torch.cuda.is_available():
return n.cuda()
print(n.size() + "")
return n
def image_tensor(path):
img = Image.open(path)
trans = transforms.ToTensor()
n = Variable(compose(trans(img)))
n.cpu()
if torch.cuda.is_available():
return n.cuda()
return n
def fake_data_target(size):
'''
Tensor containing zeros, with shape = size
'''
print("fake target")
data = Variable(torch.zeros(size, 1).cuda())
data.cpu()
if torch.cuda.is_available(): return data.cuda()
return data
def real_data_target(size):
'''
Tensor containing ones, with shape = size
'''
print("real target")
data = Variable(torch.ones(size, 1).cuda())
data.cpu()
if torch.cuda.is_available(): return data.cuda()
return data
def image_tensor(path, size):
img = Image.open(path)
n = Variable(compose(img))
n.resize_(size, 100)
n.cpu()
print(str(n.size()) + " image seed size")
if torch.cuda.is_available():
return n.cuda()
return n
def __process_predict(val_loader, model, criterion):
losses = AverageMeter()
losses_lin = AverageMeter()
model.eval()
prediction_list = list()
act_list = list()
for local_batch, local_labels in val_loader:
image_face = (local_batch[0]).to(DEVICE).permute(0, 3, 1,
2).float().cuda()
image_left_eye = (local_batch[1]).to(DEVICE).permute(
0, 3, 1, 2).float().cuda()
image_right_eye = (local_batch[2]).to(DEVICE).permute(
0, 3, 1, 2).float().cuda()
face_grid = (local_batch[3]).to(DEVICE).float().cuda()
gaze = torch.t(torch.stack(local_labels).to(DEVICE).float()).cuda()
# image_face = (local_batch[0]).to(DEVICE).permute(0, 3, 1, 2).float()
# image_left_eye = (local_batch[1]).to(DEVICE).permute(0, 3, 1, 2).float()
# image_right_eye = (local_batch[2]).to(DEVICE).permute(0, 3, 1, 2).float()
# face_grid = (local_batch[3]).to(DEVICE).float()
# gaze = torch.t(torch.stack(local_labels).to(DEVICE).float())
image_face = Variable(image_face, requires_grad=False)
image_left_eye = Variable(image_left_eye, requires_grad=False)
image_right_eye = Variable(image_right_eye, requires_grad=False)
face_grid = Variable(face_grid, requires_grad=False)
gaze = Variable(gaze, requires_grad=False)
with torch.no_grad():
output = model(image_face, image_left_eye, image_right_eye,
face_grid)
loss = criterion(output, gaze)
act_list.append(gaze.cpu().numpy())
prediction_list.append(output.cpu().numpy())
loss_lin = output - gaze
loss_lin = torch.mul(loss_lin, loss_lin)
loss_lin = torch.sum(loss_lin, 1)
loss_lin = torch.mean(torch.sqrt(loss_lin))
losses.update(loss.data.item(), image_face.size(0))
losses_lin.update(loss_lin.item(), image_face.size(0))
predictions = np.concatenate(prediction_list)
y = np.concatenate(act_list)
return predictions, y
def extract_feature(val_loader, model):
feature_extraction_model = copy.deepcopy(model)
new_fc = torch.nn.Sequential(*list(model.fc.children())[:-1])
feature_extraction_model.fc = new_fc
# switch to evaluate mode
model.eval()
feature_list = list()
act_list = list()
for local_batch, local_labels in val_loader:
image_face = (local_batch[0]).to(DEVICE).permute(0, 3, 1,
2).float().cuda()
image_left_eye = (local_batch[1]).to(DEVICE).permute(0, 3, 1,
2).float().cuda()
image_right_eye = (local_batch[2]).to(DEVICE).permute(
0, 3, 1, 2).float().cuda()
face_grid = (local_batch[3]).to(DEVICE).float().cuda()
gaze = torch.t(torch.stack(local_labels).to(DEVICE).float()).cuda()
# imFace = (local_batch[0]).to(DEVICE).permute(0, 3, 1, 2).float()
# imEyeL = (local_batch[1]).to(DEVICE).permute(0, 3, 1, 2).float()
# imEyeR = (local_batch[2]).to(DEVICE).permute(0, 3, 1, 2).float()
# faceGrid = (local_batch[3]).to(DEVICE).float()
# gaze = torch.t(torch.stack(local_labels).to(DEVICE).float())
image_face = Variable(image_face, requires_grad=False)
image_left_eye = Variable(image_left_eye, requires_grad=False)
image_right_eye = Variable(image_right_eye, requires_grad=False)
face_grid = Variable(face_grid, requires_grad=False)
gaze = Variable(gaze, requires_grad=False)
# compute output
with torch.no_grad():
output = feature_extraction_model(image_face, image_left_eye,
image_right_eye, face_grid)
feature_list.append(output.cpu().numpy())
act_list.append(gaze.cpu().numpy())
features = np.concatenate(feature_list)
y = np.concatenate(act_list)
return features, y
D.cuda()
G_criterionLoss = GeneratorLoss().cuda()
D_criterionloss = nn.BCEWithLogitsLoss()
G.eval()
D.eval()
with torch.no_grad():
for ix,(lr1, lr2, lr3, name) in enumerate(test_loader):
y_vl_dim=lr1.size()[-1]
if(lr1.size()[-1]!=lr2.size()[-1] or lr1.size()[-1]!=lr3.size()[-1]):
continue
lr_batched=np.array([np.dstack((lr1.detach().tolist()[0],lr2.detach().tolist()[0],lr3.detach().tolist()[0]))])
lr_batched=np.array(lr_batched)
lr_batched=torch.from_numpy(lr_batched).float()
val_z=Variable(lr_batched.permute(0,3,1,2))
val_z=val_z.cuda()
sr_test = G(val_z)
output_map=sr_test.cpu().data.numpy()
ccm_map=val_z.cpu().data.numpy()
if not os.path.exists(os.path.join('./data/output_dir/')):
os.mkdir('./data/output_dir/')
np.save(os.path.join('./data/output_dir/Original_','{}.npy'.format(name[0])), ccm_map)
np.save(os.path.join('./data/output_dir/ContactGAN_','{}.npy'.format(name[0])), output_map)
def test(model,
test_generator,
total_size,
loss,
input_channels,
confusion_matrix=None,
conv=True,
one_trace=False,
verbose=False,
elec=False,
output_size=2,
exo=False,
three_convs=False,
clf=[]):
'''
Function to compute results in test set given a model.
@model: model to use
'''
total_loss = []
correct = 0
total = 0
cm = meter.ConfusionMeter(output_size)
with torch.no_grad():
for values in tqdm(test_generator):
if (conv and one_trace and not three_convs):
if (elec):
inp = Variable(values[0]).view(values[0].size(0), 1,
values[0].size(1),
values[0].size(2))
# inp = Variable(values[0]).view(values[0].size(0), 1, values[0].size(1))
inp = inp.cuda()
else:
inp = Variable(values[0]).view(values[0].size(0), 1,
values[0].size(1))
elif (conv and not three_convs):
inp = Variable(values[0]).view(values[0].size(0),
values[0].size(1),
values[0].size(2))
inp = inp.cuda()
elif (conv and three_convs):
inp_tank1 = Variable(values[0]).view(values[0].size(0), 1,
values[0].size(1),
values[0].size(2))
inp_tank2 = Variable(values[1]).view(values[1].size(0), 1,
values[0].size(1),
values[1].size(2))
inp_tank3 = Variable(values[2]).view(values[2].size(0), 1,
values[0].size(1),
values[2].size(2))
inp_tank1 = inp_tank1.cuda()
inp_tank2 = inp_tank2.cuda()
inp_tank3 = inp_tank3.cuda()
else:
inp = Variable(values[0])
if (exo):
if (one_trace):
inp_exo = Variable(values[1]).view(values[1].size(0),
values[1].size(1))
output = Variable(values[2].squeeze()).type(
torch.LongTensor)
inp_exo = inp_exo.cuda()
output = output.cuda()
else:
if (not three_convs):
inp_exo = Variable(values[1]).view(
values[1].size(0), values[1].size(1))
output = Variable(values[2].squeeze()).type(
torch.LongTensor)
inp_exo = inp_exo.cuda()
output = output.cuda()
else:
inp_exo = []
for value in values[3]:
aux = Variable(value).view(value.size(0),
value.size(1))
aux = aux.cuda()
inp_exo.append(aux)
output = Variable(values[4].squeeze()).type(
torch.LongTensor)
output = output.cuda()
else:
if (not three_convs):
output = Variable(values[1].squeeze()).type(
torch.LongTensor)
output = output.cuda()
else:
output = Variable(values[3].squeeze()).type(
torch.LongTensor)
output = output.cuda()
if (clf != []):
usesvm = True
else:
usesvm = False
if (exo):
if (not three_convs):
predicted = model(inp, inp_exo)
else:
predicted = model(inp_tank1, inp_tank2, inp_tank3, inp_exo,
usesvm)
elif (not three_convs):
if (usesvm):
predicted = model(inp)
else:
predicted = model(inp, usesvm)
else:
predicted = model(inp_tank1, inp_tank2, inp_tank3, svm_=usesvm)
if (clf != []):
predicted_cpu = predicted.cpu().detach().numpy()
output_cpu = output.cpu().detach().numpy()
predicted = clf.predict(predicted_cpu)
if (cm is not None):
try:
cm.add(
torch.tensor(predicted).data.squeeze(),
output.type(torch.LongTensor))
except:
cm.add(predicted.data,
output.unsqueeze(0).type(torch.LongTensor))
# softmax to get the probability of the classes
# predicted = F.softmax(predicted, dim=1)
try:
# total_loss.append(loss(predicted, output))
computed_loss = np.abs(predicted - output_cpu)
computed_loss = np.asarray(
np.sum(computed_loss) / computed_loss.shape[0])
total_loss.append(computed_loss)
except:
total_loss.append(loss(predicted, output.unsqueeze(0)))
try:
total += output.size(0)
except:
total += 1
# correct += (labels == output).sum().item()
correct += (predicted == output_cpu).sum().item()
else:
_, labels = torch.max(predicted.data, 1)
if (cm is not None):
try:
cm.add(predicted.data.squeeze(),
output.type(torch.LongTensor))
except:
cm.add(predicted.data,
output.unsqueeze(0).type(torch.LongTensor))
try:
total_loss.append(loss(predicted, output))
except:
total_loss.append(loss(predicted, output.unsqueeze(0)))
try:
total += output.size(0)
except:
total += 1
correct += (labels == output).sum().item()
if (verbose):
print(
'Mean and standard deviation in dataset with size {} are: {} +- {}'
.format(total_size, np.mean(total_loss), np.std(total_loss)))
try:
return np.mean(total_loss), (100 * correct) / total, cm
except:
return torch.mean(torch.stack(total_loss),
dim=0), (100 * correct) / total, cm
def train(model, training_generator, n_epochs, total_size, loss, optimizer,
val_generator, val_size, conv, one_trace, verbose, elec, output_size,
exo, three_convs, input_channels, usesvm):
# Loss and optimizer
best_model = model
i = 0
validation_losses = []
training_losses = []
validation_accuracy = []
training_accuracy = []
cms = []
clf = []
for epoch in range(n_epochs):
print("Epoch {}/{}".format(epoch, n_epochs))
batch_loss = []
correct = 0
total = 0
stop = 0
for values in tqdm(training_generator):
if (conv and one_trace and not three_convs):
if (elec):
inp = Variable(values[0]).view(values[0].size(0), 1,
values[0].size(1),
values[0].size(2))
# inp = Variable(values[0]).view(values[0].size(0), 1, values[0].size(1))
inp = inp.cuda()
else:
inp = Variable(values[0]).view(values[0].size(0), 1,
values[0].size(1))
elif (conv and not three_convs):
inp = Variable(values[0]).view(values[0].size(0),
values[0].size(1),
values[0].size(2))
inp = inp.cuda()
elif (conv and three_convs):
inp_tank1 = Variable(values[0]).view(values[0].size(0), 1,
values[0].size(1),
values[0].size(2))
inp_tank2 = Variable(values[1]).view(values[1].size(0), 1,
values[0].size(1),
values[1].size(2))
inp_tank3 = Variable(values[2]).view(values[2].size(0), 1,
values[0].size(1),
values[2].size(2))
inp_tank1 = inp_tank1.cuda()
inp_tank2 = inp_tank2.cuda()
inp_tank3 = inp_tank3.cuda()
else:
inp = Variable(values[0])
if (exo):
if (not three_convs):
inp_exo = Variable(values[1]).view(values[1].size(0),
values[1].size(1))
output = Variable(values[2].squeeze()).type(
torch.LongTensor)
inp_exo = inp_exo.cuda()
output = output.cuda()
else:
inp_exo = []
for value in values[3]:
aux = Variable(value).view(value.size(0),
value.size(1))
aux = aux.cuda()
inp_exo.append(aux)
output = Variable(values[4].squeeze()).type(
torch.LongTensor)
output = output.cuda()
else:
if (not three_convs):
output = Variable(values[1].squeeze()).type(
torch.LongTensor)
output = output.cuda()
else:
output = Variable(values[3].squeeze()).type(
torch.LongTensor)
output = output.cuda()
# Forward pass
if (exo):
if (not three_convs):
predicted = model(inp, inp_exo)
else:
predicted = model(inp_tank1, inp_tank2, inp_tank3, inp_exo,
usesvm)
elif (not three_convs):
if (usesvm):
predicted = model(inp)
else:
predicted = model(inp, usesvm)
else:
predicted = model(inp_tank1, inp_tank2, inp_tank3, svm_=usesvm)
if (usesvm):
predicted_cpu = predicted.cpu().detach().numpy()
output_cpu = output.cpu().detach().numpy()
if (np.unique(output_cpu).shape[0] > 1):
svm_ = svm.SVC(kernel='rbf', gamma='scale')
if (not one_trace):
clf = GridSearchCV(svm_, parameters, cv=5)
else:
clf = svm_
clf.fit(predicted_cpu, output_cpu)
predicted = clf.predict(predicted_cpu)
# softmax to get the probability of the classes
# predicted = F.softmax(predicted, dim=1)
# _, labels = torch.max(predicted.data, 1)
try:
total += output.size(0)
except:
total += 1
correct += (predicted == output_cpu).sum().item()
# correct += (labels == output).sum().item()
try:
computed_loss = np.abs(predicted - output_cpu)
computed_loss = np.asarray(
np.sum(computed_loss) / computed_loss.shape[0])
computed_loss = torch.tensor(computed_loss,
requires_grad=True)
# computed_loss = loss(predicted, output)
except:
computed_loss = loss(predicted, output.unsqueeze(0))
batch_loss.append(computed_loss.item())
else:
# softmax to get the probability of the classes
# predicted = F.softmax(predicted, dim=1)
_, labels = torch.max(predicted.data, 1)
try:
total += output.size(0)
except:
total += 1
correct += (labels == output).sum().item()
try:
computed_loss = loss(predicted, output)
except:
computed_loss = loss(predicted, output.unsqueeze(0))
batch_loss.append(computed_loss.item())
# Backward and optimize
optimizer.zero_grad()
computed_loss.backward()
optimizer.step()
stop += 1
training_losses.append(np.mean(batch_loss))
model = model.eval()
i += 1
confusion_matrix = meter.ConfusionMeter(
output_size) # I have 5 classes here
val_loss, accuracy, confusion_matrix = validation(
model,
val_generator,
val_size,
loss,
confusion_matrix=confusion_matrix,
conv=conv,
one_trace=one_trace,
verbose=verbose,
elec=elec,
exo=exo,
three_convs=three_convs,
input_channels=input_channels,
clf=clf)
model = model.train()
validation_losses.append(val_loss)
print('Training: Epoch [{}/{}], Loss: {:.4f}, Acc: {:.4f}'.format(
epoch + 1, n_epochs, computed_loss.item(),
(100 * correct) / total))
print('Val: Epoch [{}/{}], Loss: {:.4f}, Acc: {:.4f}'.format(
epoch + 1, n_epochs, val_loss, accuracy))
validation_accuracy.append(accuracy)
cms.append(confusion_matrix)
training_accuracy.append((100 * correct) / total)
results = {
"validation_losses": validation_losses,
"training_losses": training_losses,
"validation_accuracy": validation_accuracy,
"training_accuracy": training_accuracy,
"confusion_matrix": cms
}
return model.eval(), clf, results
one_hot_labels = Variable(
torch.from_numpy(one_hot_labels)).cuda()
data = Variable(torch.from_numpy(data)).cuda()
labels = Variable(torch.from_numpy(labels)).cuda()
outputs = imitate_net([data, one_hot_labels, k])
loss_k = (losses_joint(outputs, labels, time_steps=k + 1) / (
k + 1)) / len(data_labels_paths.keys()) / config.num_traj
loss_k.backward()
loss += loss_k.data
del loss_k
optimizer.step()
train_loss += loss
log_value('train_loss_batch',
loss.cpu().numpy(),
epoch * (config.train_size //
(config.batch_size * config.num_traj)) + batch_idx)
mean_train_loss = train_loss / (config.train_size // (config.batch_size))
log_value('train_loss', mean_train_loss.cpu().numpy(), epoch)
imitate_net.eval()
loss = Variable(torch.zeros(1)).cuda()
metrics = {"cos": 0, "iou": 0, "cd": 0}
IOU = 0
COS = 0
CD = 0
for batch_idx in range(config.test_size // (config.batch_size)):
parser = ParseModelOutput(generator.unique_draw, max_len // 2 + 1, max_len,
config.canvas_shape)
for k in data_labels_paths.keys():
val_target_batched = [hr.detach().tolist()]
val_target_batched=np.array(val_target_batched)
val_target_batched=torch.from_numpy(val_target_batched).float()
val_target=Variable(val_target_batched)
val_target=val_target.cuda()
hr_test=D(val_target).mean()
hr_fake=D(sr_test).mean()
G_loss_valid=G_criterionLoss(hr_fake, sr_test, val_target)
D_loss_valid=1 - hr_test + hr_fake
# D_loss_valid = D_criterionloss(hr_fake,hr_test)
# print(torch.cuda.memory_allocated())
true_map=val_target.cpu().data.numpy()
output_map=sr_test.cpu().data.numpy()
ccm_map=val_z.cpu().data.numpy()
gan_accuracy_info=evaluate(true_map[0],output_map[0])
ccm_accuracy_info=evaluate(true_map[0],ccm_map[0])
gan_accuracy[ix] = gan_accuracy_info
ccm_accuracy[ix] = ccm_accuracy_info
for r in [10, 5, 2, 1]:
for type in ['short', 'medium', 'long']:
top_l_r = str('L/{}'.format(r))
score = np.average([info[top_l_r][type] for info in gan_accuracy.values()])
ccmScore=np.average([info[top_l_r][type] for info in ccm_accuracy.values()])
print('ContactGAN - For {} and {}-range contacts: {}'.format(top_l_r, type, score),flush=True)
print('METHOD - For {} and {}-range contacts: {}'.format(top_l_r, type, ccmScore),flush=True)
def val_epoch(cfg, epoch, model, writer, use_cuda, args):
print('validation at epoch {}'.format(epoch))
num_gpus = len(args.gpu.split(','))
model.eval()
video_list = [line.rstrip().replace('.txt', '') for line in open(cfg.test_list, 'r').readlines()]
if args.input_type == 'rgb':
assert os.path.exists(cfg.rgb_test_file)
data = h5.File(cfg.rgb_test_file, 'r')
elif args.input_type == 'flow':
assert os.path.exists(cfg.flow_test_file)
data = h5.File(cfg.flow_test_file, 'r')
elif args.input_type == 'combined':
assert os.path.exists(cfg.combined_test_file)
data = h5.File(cfg.combined_test_file, 'r')
initial_predictions, predictions, ground_truth = [], [], []
for i, video in tqdm(enumerate(video_list)):
features = data[video]
labels = build_labels(video, cfg.annotations_file, len(features), cfg.num_classes, args.add_background)
if args.add_background:
num_classes = cfg.num_classes + 1
else:
num_classes = cfg.num_classes
features = np.array(features)
labels = np.array(labels)
features = Variable(torch.from_numpy(features).type(torch.FloatTensor))
labels = Variable(torch.from_numpy(labels).type(torch.FloatTensor))
assert len(features) == len(labels)
with torch.no_grad():
if args.num_clips > 0:
eval_mode = args.eval_mode
if len(features) < args.num_clips:
eval_mode = 'pad'
if eval_mode == 'truncate':
features = features[0:len(features) - (len(features) % args.num_clips)]
labels = labels[0:len(labels) - (len(labels) % args.num_clips)]
features = torch.stack([features[i:i + args.num_clips] for i in range(0, len(features), args.num_clips)])
labels = torch.stack([labels[i:i + args.num_clips] for i in range(0, len(labels), args.num_clips)])
elif eval_mode == 'pad':
features_to_append = torch.zeros(args.num_clips - len(features) % args.num_clips, features.shape[1])
labels_to_append = torch.zeros(args.num_clips - len(labels) % args.num_clips, labels.shape[1])
features = torch.cat((features, features_to_append), 0)
labels = torch.cat((labels, labels_to_append), 0)
features = torch.stack([features[i:i + args.num_clips] for i in range(0, len(features), args.num_clips)])
labels = torch.stack([labels[i:i + args.num_clips] for i in range(0, len(labels), args.num_clips)])
elif eval_mode == 'slide':
slide_rate = 16
features_to_append = torch.zeros(slide_rate - len(features) % slide_rate, features.shape[-1])
labels_to_append = torch.zeros(slide_rate - len(labels) % slide_rate, labels.shape[-1])
features = torch.cat((features, features_to_append), 0)
labels = torch.cat((labels, labels_to_append), 0)
features = torch.stack([features[i:i + args.num_clips] for i in range(0, len(features) - args.num_clips + 1, slide_rate)])
labels = torch.stack([labels[i:i + args.num_clips] for i in range(0, len(labels) - args.num_clips + 1, slide_rate)])
assert len(features) > 0
else:
features = torch.unsqueeze(features, 0)
labels = torch.unsqueeze(labels, 0)
features = features.cuda()
labels = labels.cuda()
out = model(features)
outputs = out['final_output']
initial = out['init_output']
outputs = nn.Sigmoid()(outputs)
initial = nn.Sigmoid()(initial)
outputs = outputs.reshape(-1, num_classes)
initial = initial.reshape(-1, num_classes)
labels = labels.reshape(-1, num_classes)
outputs = outputs.cpu().data.numpy()
initial = initial.cpu().data.numpy()
labels = labels.cpu().data.numpy()
assert len(outputs) == len(labels)
predictions.extend(outputs)
ground_truth.extend(labels)
initial_predictions.extend(initial)
ground_truth = np.array(ground_truth)
predictions = np.array(predictions)
initial_predictions = np.array(initial_predictions)
avg_precision_score = average_precision_score(ground_truth, predictions, average=None)
initial_avg_precision_score = average_precision_score(ground_truth, initial_predictions, average=None)
predictions = (np.array(predictions) > args.f1_threshold).astype(int)
ground_truth = (np.array(ground_truth) > args.f1_threshold).astype(int)
results_actions = precision_recall_fscore_support(np.array(ground_truth), np.array(predictions), average=None)
f1_scores, precision, recall = results_actions[2], results_actions[0], results_actions[1]
if args.add_background:
avg_precision_score = avg_precision_score[:-1]
initial_avg_precision_score = initial_avg_precision_score[:-1]
f1_scores = f1_score[:-1]
print('Validation Epoch: %d, F1-Score: %s' % (epoch, str(f1_scores)), flush=True)
print('Validation Epoch: %d, Average Precision: %s' % (epoch, str(avg_precision_score)), flush=True)
print('Validation Epoch: %d, F1-Score: %4f, mAP: %4f, initial mAP: %4f'
% (epoch, np.nanmean(f1_scores), np.nanmean(avg_precision_score), np.nanmean(initial_avg_precision_score)), flush=True)
writer.add_scalar('Validation F1 Score', np.nanmean(f1_scores), epoch)
writer.add_scalar('Validation Precision', np.nanmean(precision), epoch)
writer.add_scalar('Validation Recall', np.nanmean(recall), epoch)
writer.add_scalar('Validation AP', np.nanmean(avg_precision_score), epoch)
return np.nanmean(avg_precision_score)
