def evaluate(args, loader, generator):
total_traj = 0
predictions = []
with torch.no_grad():
for batch in loader:
batch = [tensor.cuda() for tensor in batch]
(
obs_traj,
pred_traj_gt,
obs_traj_rel,
pred_traj_gt_rel,
non_linear_ped,
loss_mask,
seq_start_end,
) = batch
total_traj += pred_traj_gt.size(1)
for _ in range(args.num_samples):
pred_traj_fake_rel = generator(obs_traj_rel, obs_traj,
seq_start_end, 0, 3)
pred_traj_fake_rel = pred_traj_fake_rel[-args.pred_len:]
pred_traj_fake = relative_to_abs(pred_traj_fake_rel,
obs_traj[-1])
predictions.append(pred_traj_fake)
return predictions
def evaluate(args, loader, generator, num_samples):
ade_outer, fde_outer = [], []
total_traj = 0
with torch.no_grad():
for batch in loader:
batch = [tensor.cuda() for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel,
non_linear_ped, loss_mask, seq_start_end) = batch
ade, fde = [], []
total_traj += pred_traj_gt.size(1)
for _ in range(num_samples):
pred_traj_fake_rel = generator(obs_traj, obs_traj_rel,
seq_start_end)
pred_traj_fake = relative_to_abs(pred_traj_fake_rel,
obs_traj[-1])
ade.append(
displacement_error(pred_traj_fake,
pred_traj_gt,
mode='raw'))
fde.append(
final_displacement_error(pred_traj_fake[-1],
pred_traj_gt[-1],
mode='raw'))
ade_sum = evaluate_helper(ade, seq_start_end)
fde_sum = evaluate_helper(fde, seq_start_end)
ade_outer.append(ade_sum)
fde_outer.append(fde_sum)
ade = sum(ade_outer) / (total_traj * args.pred_len)
fde = sum(fde_outer) / (total_traj)
return ade, fde
def discriminator_step(args, batch, generator, discriminator, d_loss_fn,
optimizer_d):
batch = [tensor.cuda() for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, non_linear_ped,
loss_mask, seq_start_end) = batch
losses = {}
loss = torch.zeros(1).to(pred_traj_gt)
generator_out = generator(obs_traj, obs_traj_rel, seq_start_end)
pred_traj_fake_rel = generator_out
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
traj_real = torch.cat([obs_traj, pred_traj_gt], dim=0)
traj_real_rel = torch.cat([obs_traj_rel, pred_traj_gt_rel], dim=0)
traj_fake = torch.cat([obs_traj, pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel, pred_traj_fake_rel], dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel, seq_start_end)
scores_real = discriminator(traj_real, traj_real_rel, seq_start_end)
# Compute loss with optional gradient penalty
data_loss = d_loss_fn(scores_real, scores_fake)
losses['D_data_loss'] = data_loss.item()
loss += data_loss
losses['D_total_loss'] = loss.item()
optimizer_d.zero_grad()
loss.backward()
if args.clipping_threshold_d > 0:
nn.utils.clip_grad_norm_(discriminator.parameters(),
args.clipping_threshold_d)
optimizer_d.step()
return losses
def discriminator_step(batch, generator, discriminator, d_loss_fn,
optimizer_d):
batch = [tensor.cuda() for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, vgg_list) = batch
losses = {}
loss = torch.zeros(1).to(pred_traj_gt)
generator_out = generator(obs_traj, obs_traj_rel, vgg_list)
pred_traj_fake_rel = generator_out
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1, :, 0, :])
traj_real = torch.cat([obs_traj[:, :, 0, :], pred_traj_gt], dim=0)
traj_real_rel = torch.cat([obs_traj_rel[:, :, 0, :], pred_traj_gt_rel],
dim=0)
traj_fake = torch.cat([obs_traj[:, :, 0, :], pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel[:, :, 0, :], pred_traj_fake_rel],
dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel)
scores_real = discriminator(traj_real, traj_real_rel)
data_loss = d_loss_fn(scores_real, scores_fake)
losses['D_data_loss'] = data_loss.item()
loss += data_loss
losses['D_total_loss'] = loss.item()
optimizer_d.zero_grad()
loss.backward()
optimizer_d.step()
return losses
def evaluate(loader, generator):
ade_outer, fde_outer = [], []
total_traj = 0
with torch.no_grad():
for batch in loader:
batch = [tensor.cuda() for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, vgg_list) = batch
ade, fde = [], []
total_traj += pred_traj_gt.size(1)
for _ in range(NUM_SAMPLES):
pred_traj_fake_rel = generator(obs_traj, obs_traj_rel, vgg_list)
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1, :, 0, :])
ade.append(displacement_error(pred_traj_fake, pred_traj_gt, mode='raw'))
fde.append(final_displacement_error(pred_traj_fake[-1], pred_traj_gt[-1], mode='raw'))
ade_sum = evaluate_helper(ade)
fde_sum = evaluate_helper(fde)
ade_outer.append(ade_sum)
fde_outer.append(fde_sum)
ade = sum(ade_outer) / (total_traj * PRED_LEN)
fde = sum(fde_outer) / (total_traj)
return ade, fde
def forward(self,
obs_traj,
obs_traj_rel,
seq_start_end,
obs_ped_speed,
pred_ped_speed,
pred_traj,
train_or_test,
speed_to_add,
user_noise=None):
batch = obs_traj_rel.size(1)
final_encoder_h = self.encoder(obs_traj_rel, obs_ped_speed)
if POOLING_TYPE:
if train_or_test == 1:
simulated_ped_speed = speed_control(pred_ped_speed[0, :, :],
SPEED_TO_ADD,
seq_start_end)
next_speed = simulated_ped_speed
else:
next_speed = pred_ped_speed[0, :, :]
sspm = self.social_speed_pooling(final_encoder_h, seq_start_end,
train_or_test, speed_to_add,
obs_traj[-1, :, :], next_speed)
mlp_decoder_context_input = torch.cat(
[final_encoder_h.view(-1, self.h_dim), sspm], dim=1)
else:
mlp_decoder_context_input = final_encoder_h.view(-1, self.h_dim)
noise_input = self.mlp_decoder_context(mlp_decoder_context_input)
decoder_h = self.add_noise(noise_input, seq_start_end).unsqueeze(dim=0)
if USE_GPU:
decoder_c = torch.zeros(self.num_layers, batch, self.h_dim).cuda()
else:
decoder_c = torch.zeros(self.num_layers, batch, self.h_dim)
state_tuple = (decoder_h, decoder_c)
decoder_out = self.decoder(obs_traj[-1], obs_traj_rel[-1], state_tuple,
seq_start_end, speed_to_add, pred_ped_speed,
train_or_test)
pred_traj_fake_rel, final_decoder_h = decoder_out
# LOGGING THE OUTPUT OF ALL SEQUENCES TO TEST THE SPEED AND TRAJECTORIES
if train_or_test == 1:
simulated_trajectories = []
for _, (start, end) in enumerate(seq_start_end):
start = start.item()
end = end.item()
obs_test_traj = obs_traj[:, start:end, :]
pred_test_traj_rel = pred_traj_fake_rel[:, start:end, :]
pred_test_traj = relative_to_abs(pred_test_traj_rel,
obs_test_traj[-1])
speed_added = pred_ped_speed[0, start:end, :]
simulated_trajectories.append(pred_test_traj)
return pred_traj_fake_rel
def generator_step(args, batch, generator, discriminator, g_loss_fn,
optimizer_g, discriminator_wight):
if args.use_gpu == 1:
batch = [tensor.cuda() for tensor in batch]
else:
batch = [tensor for tensor in batch]
# batch = [tensor.cuda() for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, non_linear_ped,
loss_mask, seq_start_end) = batch
losses = {}
loss = torch.zeros(1).to(pred_traj_gt)
g_l2_loss_rel = []
loss_mask = loss_mask[:, args.obs_len:]
for _ in range(args.best_k):
generator_out = generator(obs_traj, obs_traj_rel, seq_start_end)
pred_traj_fake_rel = generator_out
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
if args.l2_loss_weight > 0:
g_l2_loss_rel.append(args.l2_loss_weight * l2_loss(
pred_traj_fake_rel, pred_traj_gt_rel, loss_mask, mode='raw'))
g_l2_loss_sum_rel = torch.zeros(1).to(pred_traj_gt)
if args.l2_loss_weight > 0:
g_l2_loss_rel = torch.stack(g_l2_loss_rel, dim=1)
for start, end in seq_start_end.data:
_g_l2_loss_rel = g_l2_loss_rel[start:end]
_g_l2_loss_rel = torch.sum(_g_l2_loss_rel, dim=0)
_g_l2_loss_rel = torch.min(_g_l2_loss_rel) / torch.sum(
loss_mask[start:end])
g_l2_loss_sum_rel += _g_l2_loss_rel
losses['G_l2_loss_rel'] = g_l2_loss_sum_rel.item()
loss += g_l2_loss_sum_rel
traj_fake = torch.cat([obs_traj, pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel, pred_traj_fake_rel], dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel, seq_start_end)
discriminator_loss = g_loss_fn(scores_fake)
loss += (discriminator_wight * discriminator_loss)
losses['G_discriminator_loss'] = discriminator_loss.item()
losses['G_total_loss'] = loss.item()
optimizer_g.zero_grad()
loss.backward()
if args.clipping_threshold_g > 0:
nn.utils.clip_grad_norm_(generator.parameters(),
args.clipping_threshold_g)
optimizer_g.step()
return losses
def generator_step(batch, generator, discriminator, g_loss_fn, optimizer_g):
"""This step is similar to Social GAN Code"""
if USE_GPU:
batch = [tensor.cuda() for tensor in batch]
else:
batch = [tensor for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, loss_mask,
seq_start_end, obs_ped_speed, pred_ped_speed) = batch
losses = {}
loss = torch.zeros(1).to(pred_traj_gt)
g_l2_loss_rel = []
loss_mask = loss_mask[:, OBS_LEN:]
for _ in range(BEST_K):
generator_out = generator(obs_traj, obs_traj_rel, seq_start_end,
obs_ped_speed, pred_ped_speed, pred_traj_gt,
TRAIN_METRIC, SPEED_TO_ADD)
pred_traj_fake_rel = generator_out
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
if L2_LOSS_WEIGHT > 0:
g_l2_loss_rel.append(L2_LOSS_WEIGHT * l2_loss(
pred_traj_fake_rel, pred_traj_gt_rel, loss_mask, mode='raw'))
g_l2_loss_sum_rel = torch.zeros(1).to(pred_traj_gt)
if L2_LOSS_WEIGHT > 0:
g_l2_loss_rel = torch.stack(g_l2_loss_rel, dim=1)
for start, end in seq_start_end.data:
_g_l2_loss_rel = g_l2_loss_rel[start:end]
_g_l2_loss_rel = torch.sum(_g_l2_loss_rel, dim=0)
_g_l2_loss_rel = torch.min(_g_l2_loss_rel) / torch.sum(
loss_mask[start:end])
g_l2_loss_sum_rel += _g_l2_loss_rel
losses['G_l2_loss_rel'] = g_l2_loss_sum_rel.item()
loss += g_l2_loss_sum_rel
traj_fake = torch.cat([obs_traj, pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel, pred_traj_fake_rel], dim=0)
ped_speed = torch.cat([obs_ped_speed, pred_ped_speed], dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel, ped_speed,
seq_start_end)
discriminator_loss = g_loss_fn(scores_fake)
loss += discriminator_loss
losses['G_discriminator_loss'] = discriminator_loss.item()
losses['G_total_loss'] = loss.item()
optimizer_g.zero_grad()
loss.backward()
optimizer_g.step()
return losses
def evaluate(args, loader, generator, num_samples):
ade_outer, fde_outer = [], []
total_traj = 0
trajs = []
with torch.no_grad():
for batch in loader:
# batch = [tensor.cuda() for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel,
non_linear_ped, loss_mask, seq_start_end) = batch
obs_traj = obs_traj.float()
obs_traj_rel = obs_traj_rel.float()
pred_traj_gt = pred_traj_gt.double()
ade, fde = [], []
total_traj += pred_traj_gt.size(1)
for _ in range(num_samples):
pred_traj_fake_rel = generator(obs_traj, obs_traj_rel,
seq_start_end)
pred_traj_fake = relative_to_abs(pred_traj_fake_rel,
obs_traj[-1])
pred_traj_fake = pred_traj_fake.double()
trajs.append([
obs_traj.cpu().numpy(),
pred_traj_fake.cpu().numpy(),
pred_traj_gt.cpu().numpy(),
seq_start_end.cpu().numpy()
])
ade_traj = displacement_error(pred_traj_fake,
pred_traj_gt,
mode='sum')
fde_traj = final_displacement_error(pred_traj_fake[-1],
pred_traj_gt[-1],
mode='sum')
# ade.append(displacement_error(
# pred_traj_fake, pred_traj_gt, mode='sum'
# ))
# fde.append(final_displacement_error(
# pred_traj_fake[-1], pred_traj_gt[-1], mode='sum'
# ))
# ade_sum = evaluate_helper(ade, seq_start_end)
# fde_sum = evaluate_helper(fde, seq_start_end)
# ade_outer.append(ade_sum)
# fde_outer.append(fde_sum)
ade_outer.append(ade_traj)
fde_outer.append(fde_traj)
ade = sum(ade_outer) / (total_traj * args.pred_len)
fde = sum(fde_outer) / (total_traj)
return ade, fde, trajs
def generator_step(batch, generator, discriminator, g_loss_fn, optimizer_g):
batch = [tensor.cuda() for tensor in batch]
# (obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, vgg_list) = batch
# (obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel) = batch
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, n_l, l_m, V_obs,
A_obs, V_pre, A_pre, vgg_list) = batch
losses = {}
loss = torch.zeros(1).to(pred_traj_gt)
g_l2_loss_rel = []
for _ in range(BEST_K):
# generator_out = generator(obs_traj, obs_traj_rel, vgg_list)
generator_out = generator(obs_traj, obs_traj_rel, V_obs, A_obs,
vgg_list) # 生成坐标差
pred_traj_fake_rel = generator_out
pred_traj_fake = relative_to_abs(pred_traj_fake_rel,
obs_traj[0, :, :, -1]) # 12*3*2 TVC
g_l2_loss_rel.append(
l2_loss( # 生成坐标差和真实坐标差 n*1
pred_traj_fake_rel, # T V C
pred_traj_gt_rel, # N V C T
mode='raw'))
# 生成了K条轨迹并计算损失 K V
npeds = obs_traj.size(1) # V
g_l2_loss_sum_rel = torch.zeros(1).to(pred_traj_gt) # 1
g_l2_loss_rel = torch.stack(g_l2_loss_rel, dim=1) # 拼接张量 得v k
_g_l2_loss_rel = torch.sum(g_l2_loss_rel, dim=0) # 求和 得 k
_g_l2_loss_rel = torch.min(_g_l2_loss_rel) / (npeds * PRED_LEN
) # 取最小的然后取平均
g_l2_loss_sum_rel += _g_l2_loss_rel
losses['G_l2_loss_rel'] = g_l2_loss_sum_rel.item()
loss += g_l2_loss_sum_rel # 生成轨迹的损失
pred_traj_fake = pred_traj_fake.permute(1, 2, 0) # TVC——VCT
pred_traj_fake_rel = pred_traj_fake_rel.permute(1, 2, 0)
traj_fake = torch.cat([obs_traj[0], pred_traj_fake], dim=2) # VCT T=20
traj_fake_rel = torch.cat([obs_traj_rel[0], pred_traj_fake_rel], dim=2)
scores_fake = discriminator(traj_fake, traj_fake_rel) # 生成轨迹鉴别分数
discriminator_loss = g_loss_fn(scores_fake)
loss += discriminator_loss # 加入鉴别器损失
losses['G_discriminator_loss'] = discriminator_loss.item()
losses['G_total_loss'] = loss.item()
optimizer_g.zero_grad()
loss.backward()
optimizer_g.step()
return losses
def evaluate(loader, generator, num_samples):
ade_outer, fde_outer, simulated_output, total_traj, sequences = [], [], [], [], []
with torch.no_grad():
for batch in loader:
if USE_GPU:
batch = [tensor.cuda() for tensor in batch]
else:
batch = [tensor for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, loss_mask, seq_start_end, obs_ped_speed, pred_ped_speed) = batch
ade, fde, sim_op = [], [], []
total_traj.append(pred_traj_gt.size(1))
for _ in range(num_samples):
if TEST_METRIC:
pred_traj_fake_rel = generator(obs_traj, obs_traj_rel, seq_start_end, obs_ped_speed, pred_ped_speed, pred_traj_gt,
TEST_METRIC, SPEED_TO_ADD)
else:
pred_traj_fake_rel = generator(obs_traj, obs_traj_rel, seq_start_end, obs_ped_speed,
pred_ped_speed, pred_traj_gt, TEST_METRIC, SPEED_TO_ADD)
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
ade.append(displacement_error(pred_traj_fake, pred_traj_gt, mode='raw'))
fde.append(final_displacement_error(pred_traj_fake[-1], pred_traj_gt[-1], mode='raw'))
sim_op.append(pred_traj_fake)
for _, (start, end) in enumerate(seq_start_end):
num_ped = end - start
sequences.append(num_ped)
ade_outer.append(evaluate_helper(torch.stack(ade, dim=1), seq_start_end))
fde_outer.append(evaluate_helper(torch.stack(fde, dim=1), seq_start_end))
simulated_output.append(torch.cat(sim_op, dim=0))
ade = sum(ade_outer) / (sum(total_traj) * PRED_LEN)
fde = sum(fde_outer) / (sum(total_traj))
simulated_traj_for_visualization = torch.cat(simulated_output, dim=1)
sequences = torch.cumsum(torch.stack(sequences, dim=0), dim=0)
if TEST_METRIC and VERIFY_OUTPUT_SPEED:
# The speed can be verified for different sequences and this method runs for n number of batches.
verify_speed(simulated_traj_for_visualization, sequences)
if ANIMATED_VISUALIZATION_CHECK:
# Trajectories at User-defined speed for Visualization
with open('SimulatedTraj.pkl', 'wb') as f:
pickle.dump(simulated_traj_for_visualization, f, pickle.HIGHEST_PROTOCOL)
# Sequence list file used for Visualization
with open('Sequences.pkl', 'wb') as f:
pickle.dump(sequences, f, pickle.HIGHEST_PROTOCOL)
return ade, fde
def discriminator_step(batch, generator, discriminator, d_loss_fn, optimizer_d):
if USE_GPU:
batch = [tensor.cuda() for tensor in batch]
else:
batch = [tensor for tensor in batch]
if MULTI_CONDITIONAL_MODEL:
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, loss_mask, seq_start_end, obs_ped_speed, pred_ped_speed, obs_label, pred_label,
obs_obj_rel_speed) = batch
generator_out, _ = generator(obs_traj, obs_traj_rel, seq_start_end, obs_ped_speed, pred_ped_speed,
pred_traj_gt, TRAIN_METRIC, None, obs_obj_rel_speed, obs_label=obs_label, pred_label=pred_label)
else:
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, loss_mask, seq_start_end, obs_ped_speed, pred_ped_speed, obs_obj_rel_speed) = batch
generator_out, _ = generator(obs_traj, obs_traj_rel, seq_start_end, obs_ped_speed, pred_ped_speed,
pred_traj_gt, TRAIN_METRIC, None, obs_obj_rel_speed, obs_label=None, pred_label=None)
losses = {}
loss = torch.zeros(1).to(pred_traj_gt)
pred_traj_fake_rel = generator_out
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
traj_real = torch.cat([obs_traj, pred_traj_gt], dim=0)
traj_real_rel = torch.cat([obs_traj_rel, pred_traj_gt_rel], dim=0)
traj_fake = torch.cat([obs_traj, pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel, pred_traj_fake_rel], dim=0)
ped_speed = torch.cat([obs_ped_speed, pred_ped_speed], dim=0)
if MULTI_CONDITIONAL_MODEL:
label_info = torch.cat([obs_label, pred_label], dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel, ped_speed, label=label_info)
scores_real = discriminator(traj_real, traj_real_rel, ped_speed, label=label_info)
else:
scores_fake = discriminator(traj_fake, traj_fake_rel, ped_speed, label=None)
scores_real = discriminator(traj_real, traj_real_rel, ped_speed, label=None)
data_loss = d_loss_fn(scores_real, scores_fake)
losses['D_data_loss'] = data_loss.item()
loss += data_loss
losses['D_total_loss'] = loss.item()
optimizer_d.zero_grad()
loss.backward()
optimizer_d.step()
return losses
def generator_step(batch, generator, discriminator, g_loss_fn, optimizer_g):
batch = [tensor.cuda() for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, vgg_list) = batch
losses = {}
loss = torch.zeros(1).to(pred_traj_gt)
g_l2_loss_rel = []
for _ in range(BEST_K):
generator_out = generator(obs_traj, obs_traj_rel, vgg_list)
pred_traj_fake_rel = generator_out
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1, :,
0, :])
g_l2_loss_rel.append(
l2_loss(pred_traj_fake_rel, pred_traj_gt_rel, mode='raw'))
npeds = obs_traj.size(1)
g_l2_loss_sum_rel = torch.zeros(1).to(pred_traj_gt)
g_l2_loss_rel = torch.stack(g_l2_loss_rel, dim=1)
_g_l2_loss_rel = torch.sum(g_l2_loss_rel, dim=0)
_g_l2_loss_rel = torch.min(_g_l2_loss_rel) / (npeds * PRED_LEN)
g_l2_loss_sum_rel += _g_l2_loss_rel
losses['G_l2_loss_rel'] = g_l2_loss_sum_rel.item()
loss += g_l2_loss_sum_rel
traj_fake = torch.cat([obs_traj[:, :, 0, :], pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel[:, :, 0, :], pred_traj_fake_rel],
dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel)
discriminator_loss = g_loss_fn(scores_fake)
loss += discriminator_loss
losses['G_discriminator_loss'] = discriminator_loss.item()
losses['G_total_loss'] = loss.item()
optimizer_g.zero_grad()
loss.backward()
optimizer_g.step()
return losses
def discriminator_step(batch, generator, discriminator, d_loss_fn,
optimizer_d):
batch = [tensor.cuda() for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, n_l, l_m, V_obs,
A_obs, V_pre, A_pre, vgg_list) = batch
# (obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel) = batch
losses = {}
loss = torch.zeros(1).to(pred_traj_gt)
# generator_out = generator(obs_traj, obs_traj_rel, vgg_list)
generator_out = generator(obs_traj, obs_traj_rel, V_obs, A_obs, vgg_list)
pred_traj_fake_rel = generator_out
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[0, :, :,
-1]) # V C
traj_real = torch.cat([obs_traj[0], pred_traj_gt[0]],
dim=2) # 1*3*2*20 轨迹序列
traj_real_rel = torch.cat([obs_traj_rel[0], pred_traj_gt_rel[0]],
dim=2) # 1*3*2*20 轨迹差值序列 N V C T
pred_traj_fake = pred_traj_fake.permute(1, 2, 0) # T V C——V C T
# pred_traj_fake=pred_traj_fake.unsqueeze(dim=0)
pred_traj_fake_rel = pred_traj_fake_rel.permute(1, 2, 0) # T V C——V C T
# pred_traj_fake_rel=pred_traj_fake_rel.unsqueeze(dim=0)
traj_fake = torch.cat([obs_traj[0], pred_traj_fake],
dim=2) # 观测轨迹加上预测 V C T
traj_fake_rel = torch.cat([obs_traj_rel[0], pred_traj_fake_rel],
dim=2) # 观测差加上预测差 V C T
scores_fake = discriminator(traj_fake, traj_fake_rel) # 计算鉴别分数输入 VCT编码
scores_real = discriminator(traj_real, traj_real_rel) # 输入 V C T 输出 V 1
data_loss = d_loss_fn(scores_real, scores_fake) # BCE
losses['D_data_loss'] = data_loss.item()
loss += data_loss
losses['D_total_loss'] = loss.item()
optimizer_d.zero_grad()
loss.backward()
optimizer_d.step()
return losses
def discriminator_step(batch, generator, discriminator, d_loss_fn,
optimizer_d):
"""This step is similar to Social GAN Code"""
if USE_GPU:
batch = [tensor.cuda() for tensor in batch]
else:
batch = [tensor for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, loss_mask,
seq_start_end, obs_ped_speed, pred_ped_speed) = batch
losses = {}
loss = torch.zeros(1).to(pred_traj_gt)
generator_out = generator(obs_traj, obs_traj_rel, seq_start_end,
obs_ped_speed, pred_ped_speed, pred_traj_gt,
TRAIN_METRIC, SPEED_TO_ADD)
pred_traj_fake_rel = generator_out
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
traj_real = torch.cat([obs_traj, pred_traj_gt], dim=0)
traj_real_rel = torch.cat([obs_traj_rel, pred_traj_gt_rel], dim=0)
traj_fake = torch.cat([obs_traj, pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel, pred_traj_fake_rel], dim=0)
ped_speed = torch.cat([obs_ped_speed, pred_ped_speed], dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel, ped_speed,
seq_start_end)
scores_real = discriminator(traj_real, traj_real_rel, ped_speed,
seq_start_end)
data_loss = d_loss_fn(scores_real, scores_fake)
losses['D_data_loss'] = data_loss.item()
loss += data_loss
losses['D_total_loss'] = loss.item()
optimizer_d.zero_grad()
loss.backward()
optimizer_d.step()
return losses
def validate(args, model, val_loader, epoch, writer):
ade = utils.AverageMeter("ADE", ":.6f")
fde = utils.AverageMeter("FDE", ":.6f")
progress = utils.ProgressMeter(len(val_loader), [ade, fde],
prefix="Test: ")
model.eval()
with torch.no_grad():
for i, batch in enumerate(val_loader):
batch = [tensor.cuda() for tensor in batch]
(
obs_traj,
pred_traj_gt,
obs_traj_rel,
pred_traj_gt_rel,
non_linear_ped,
loss_mask,
seq_start_end,
) = batch
loss_mask = loss_mask[:, args.obs_len:]
pred_traj_fake_rel = model(obs_traj_rel, obs_traj, seq_start_end)
pred_traj_fake_rel_predpart = pred_traj_fake_rel[-args.pred_len:]
pred_traj_fake = relative_to_abs(pred_traj_fake_rel_predpart,
obs_traj[-1])
ade_, fde_ = cal_ade_fde(pred_traj_gt, pred_traj_fake)
ade_ = ade_ / (obs_traj.shape[1] * args.pred_len)
fde_ = fde_ / (obs_traj.shape[1])
ade.update(ade_, obs_traj.shape[1])
fde.update(fde_, obs_traj.shape[1])
if i % args.print_every == 0:
progress.display(i)
logging.info(" * ADE {ade.avg:.3f} FDE {fde.avg:.3f}".format(
ade=ade, fde=fde))
writer.add_scalar("val_ade", ade.avg, epoch)
return ade.avg
def evaluate(args, loader, generator):
ade_outer, fde_outer = [], []
total_traj = 0
with torch.no_grad():
for batch in loader:
batch = [tensor.cuda() for tensor in batch]
(
obs_traj,
pred_traj_gt,
obs_traj_rel,
pred_traj_gt_rel,
non_linear_ped,
loss_mask,
seq_start_end,
) = batch
ade, fde = [], []
total_traj += pred_traj_gt.size(1)
for _ in range(args.num_samples):
pred_traj_fake_rel = generator(obs_traj_rel, obs_traj,
seq_start_end, 0, 3)
pred_traj_fake_rel = pred_traj_fake_rel[-args.pred_len:]
pred_traj_fake = relative_to_abs(pred_traj_fake_rel,
obs_traj[-1])
ade_, fde_ = cal_ade_fde(pred_traj_gt, pred_traj_fake)
ade.append(ade_)
fde.append(fde_)
ade_sum = evaluate_helper(ade, seq_start_end)
fde_sum = evaluate_helper(fde, seq_start_end)
ade_outer.append(ade_sum)
fde_outer.append(fde_sum)
ade = sum(ade_outer) / (total_traj * args.pred_len)
fde = sum(fde_outer) / (total_traj)
return ade, fde
def check_accuracy(args, loader, generator, limit=False):
d_losses = []
metrics = {}
g_l2_losses_abs, g_l2_losses_rel = ([], ) * 2
disp_error, disp_error_l, disp_error_nl = ([], ) * 3
f_disp_error, f_disp_error_l, f_disp_error_nl = ([], ) * 3
total_traj, total_traj_l, total_traj_nl = 0, 0, 0
loss_mask_sum = 0
generator.eval()
with torch.no_grad():
for batch in loader:
# batch = [tensor.cuda() for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel,
non_linear_ped, loss_mask, seq_start_end) = batch
linear_ped = 1 - non_linear_ped
loss_mask = loss_mask[:, args.obs_len:]
obs_traj = obs_traj.float()
obs_traj_rel = obs_traj_rel.float()
pred_traj_fake_rel = generator(obs_traj, obs_traj_rel,
seq_start_end)
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
g_l2_loss_abs, g_l2_loss_rel = cal_l2_losses(
pred_traj_gt, pred_traj_gt_rel, pred_traj_fake,
pred_traj_fake_rel, loss_mask)
ade, ade_l, ade_nl = cal_ade(pred_traj_gt, pred_traj_fake,
linear_ped, non_linear_ped)
fde, fde_l, fde_nl = cal_fde(pred_traj_gt, pred_traj_fake,
linear_ped, non_linear_ped)
# traj_real = torch.cat([obs_traj, pred_traj_gt], dim=0)
# traj_real_rel = torch.cat([obs_traj_rel, pred_traj_gt_rel], dim=0)
# traj_fake = torch.cat([obs_traj, pred_traj_fake], dim=0)
# traj_fake_rel = torch.cat([obs_traj_rel, pred_traj_fake_rel], dim=0)
# scores_fake = discriminator(traj_fake, traj_fake_rel, seq_start_end)
# scores_real = discriminator(traj_real, traj_real_rel, seq_start_end)
# d_loss = d_loss_fn(scores_real, scores_fake)
# d_losses.append(d_loss.item())
g_l2_losses_abs.append(g_l2_loss_abs.item())
g_l2_losses_rel.append(g_l2_loss_rel.item())
disp_error.append(ade.item())
disp_error_l.append(ade_l.item())
disp_error_nl.append(ade_nl.item())
f_disp_error.append(fde.item())
f_disp_error_l.append(fde_l.item())
f_disp_error_nl.append(fde_nl.item())
loss_mask_sum += torch.numel(loss_mask.data)
total_traj += pred_traj_gt.size(1)
total_traj_l += torch.sum(linear_ped).item()
total_traj_nl += torch.sum(non_linear_ped).item()
if limit and total_traj >= args.num_samples_check:
break
# metrics['d_loss'] = sum(d_losses) / len(d_losses)
metrics['g_l2_loss_abs'] = sum(g_l2_losses_abs) / loss_mask_sum
metrics['g_l2_loss_rel'] = sum(g_l2_losses_rel) / loss_mask_sum
metrics['ade'] = sum(disp_error) / (total_traj * args.pred_len)
metrics['fde'] = sum(f_disp_error) / total_traj
if total_traj_l != 0:
metrics['ade_l'] = sum(disp_error_l) / (total_traj_l * args.pred_len)
metrics['fde_l'] = sum(f_disp_error_l) / total_traj_l
else:
metrics['ade_l'] = 0
metrics['fde_l'] = 0
if total_traj_nl != 0:
metrics['ade_nl'] = sum(disp_error_nl) / (total_traj_nl *
args.pred_len)
metrics['fde_nl'] = sum(f_disp_error_nl) / total_traj_nl
else:
metrics['ade_nl'] = 0
metrics['fde_nl'] = 0
generator.train()
return metrics
def get_trajs(frame, step=10):
'''
:param frame: last observed frame
:param step: step between each frame
:returns None if no prediction can be made, or trajs_, a dictionary containing trajectories for each pedestrian
'''
trajs_ = {}
# -1 because we include in selection
seq_range = [frame - (obs_len - 1) * step, frame + pred_len * step]
print("seq_range",seq_range)
obs_range = [frame - (obs_len - 1) * step, frame]
print("obs_range",obs_range)
raw_obs_seq = data.loc[data["frameID"].between(obs_range[0], obs_range[1], inclusive=True)]
raw_pred_seq = data.loc[data["frameID"].between(obs_range[1] + step, seq_range[1], inclusive=True)]
peds_in_seq = raw_obs_seq.pedID.unique()
curr_seq = np.zeros((len(peds_in_seq), 2, obs_len))
curr_seq_rel = np.zeros((len(peds_in_seq), 2, obs_len))
id_list = []
considered_ped = 0
for ped_id in peds_in_seq:
obs_ped_seq = raw_obs_seq.loc[raw_obs_seq.pedID == ped_id]
# seq has to have at least obs_len length
if len(obs_ped_seq.frameID) == obs_len:
id_list.append(ped_id)
pred_ped_seq = raw_pred_seq.loc[raw_pred_seq.pedID == ped_id]
trajs_[ped_id] = {}
obs_traj = obs_ped_seq[["x", "y"]].values.transpose()
obs_traj_rel = np.zeros(obs_traj.shape)
obs_traj_rel[:, 1:] = obs_traj[:, 1:] - obs_traj[:, :-1]
curr_seq[considered_ped, :, 0:obs_len] = obs_traj
curr_seq_rel[considered_ped, :, 0:obs_len] = obs_traj_rel
trajs_[ped_id]["obs"] = obs_traj.transpose()
trajs_[ped_id]["pred_gt"] = pred_ped_seq[["x", "y"]].values
considered_ped += 1
print(considered_ped)
if considered_ped > 0:
obs_list_tensor = torch.from_numpy(curr_seq[:considered_ped, :]).permute(2, 0, 1).float()#.cuda().float()
obs_list_rel_tensor = torch.from_numpy(curr_seq_rel[:considered_ped, :]).permute(2, 0, 1).float()#.cuda().float()
seq_start_end_tensor = torch.tensor([[0, considered_ped]])
for model_name, model in models.items():
if model_name=='rnn':
pred_rel = model(obs_list_tensor, obs_list_rel_tensor, seq_start_end_tensor,dat_seg)
pred_abs = relative_to_abs(pred_rel, obs_list_tensor[-1]).detach().cpu().numpy()
pred_abs_reorder = np.swapaxes(pred_abs, 0, 1)
key = "pred_" + model_name
for i in range(considered_ped):
ped_id = id_list[i]
trajs_[ped_id][key] = pred_abs_reorder[i]
if model_name=='vgg':
pred_rel = model(obs_list_tensor, obs_list_rel_tensor, seq_start_end_tensor,dat)
pred_abs = relative_to_abs(pred_rel, obs_list_tensor[-1]).detach().cpu().numpy()
pred_abs_reorder = np.swapaxes(pred_abs, 0, 1)
key = "pred_" + model_name
for k in range(considered_ped):
ped_id = id_list[k]
trajs_[ped_id][key] = pred_abs_reorder[k]
if model_name == 'sgan':
pred_rel = model(obs_list_tensor, obs_list_rel_tensor, seq_start_end_tensor)
pred_abs = relative_to_abs(pred_rel, obs_list_tensor[-1]).detach().cpu().numpy()
pred_abs_reorder = np.swapaxes(pred_abs, 0, 1)
key = "pred_" + model_name
for k in range(considered_ped):
ped_id = id_list[k]
trajs_[ped_id][key] = pred_abs_reorder[k]
if model_name == 'segnet_full':
pred_rel = model(obs_list_tensor, obs_list_rel_tensor, seq_start_end_tensor,dat_segnet_full)
pred_abs = relative_to_abs(pred_rel, obs_list_tensor[-1]).detach().cpu().numpy()
pred_abs_reorder = np.swapaxes(pred_abs, 0, 1)
key = "pred_" + model_name
for k in range(considered_ped):
ped_id = id_list[k]
trajs_[ped_id][key] = pred_abs_reorder[k]
if model_name == 'segnet':
pred_rel = model(obs_list_tensor, obs_list_rel_tensor, seq_start_end_tensor, dat_segnet)
pred_abs = relative_to_abs(pred_rel, obs_list_tensor[-1]).detach().cpu().numpy()
pred_abs_reorder = np.swapaxes(pred_abs, 0, 1)
key = "pred_" + model_name
for k in range(considered_ped):
ped_id = id_list[k]
trajs_[ped_id][key] = pred_abs_reorder[k]
return trajs_
else:
return None
def check_accuracy(loader, generator, discriminator, d_loss_fn):
d_losses = []
metrics = {}
g_l2_losses_abs, g_l2_losses_rel = ([], ) * 2
disp_error, f_disp_error, mean_speed_disp_error, final_speed_disp_error = [], [], [], []
total_traj = 0
loss_mask_sum = 0
generator.eval()
with torch.no_grad():
for batch in loader:
if USE_GPU:
batch = [tensor.cuda() for tensor in batch]
else:
batch = [tensor for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, loss_mask,
seq_start_end, obs_ped_speed, pred_ped_speed) = batch
pred_traj_fake_rel = generator(obs_traj, obs_traj_rel,
seq_start_end, obs_ped_speed,
pred_ped_speed, pred_traj_gt,
TRAIN_METRIC, SPEED_TO_ADD)
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
loss_mask = loss_mask[:, OBS_LEN:]
g_l2_loss_abs, g_l2_loss_rel = cal_l2_losses(
pred_traj_gt, pred_traj_gt_rel, pred_traj_fake,
pred_traj_fake_rel, loss_mask)
ade = displacement_error(pred_traj_gt, pred_traj_fake)
fde = final_displacement_error(pred_traj_gt, pred_traj_fake)
last_pos = obs_traj[-1]
traj_for_speed_cal = torch.cat(
[last_pos.unsqueeze(dim=0), pred_traj_fake], dim=0)
msae = cal_msae(pred_ped_speed, traj_for_speed_cal)
fse = cal_fse(pred_ped_speed[-1], pred_traj_fake)
traj_real = torch.cat([obs_traj, pred_traj_gt], dim=0)
traj_real_rel = torch.cat([obs_traj_rel, pred_traj_gt_rel], dim=0)
traj_fake = torch.cat([obs_traj, pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel, pred_traj_fake_rel],
dim=0)
ped_speed = torch.cat([obs_ped_speed, pred_ped_speed], dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel, ped_speed,
seq_start_end)
scores_real = discriminator(traj_real, traj_real_rel, ped_speed,
seq_start_end)
d_loss = d_loss_fn(scores_real, scores_fake)
d_losses.append(d_loss.item())
g_l2_losses_abs.append(g_l2_loss_abs.item())
g_l2_losses_rel.append(g_l2_loss_rel.item())
disp_error.append(ade.item())
f_disp_error.append(fde.item())
mean_speed_disp_error.append(msae.item())
final_speed_disp_error.append(fse.item())
loss_mask_sum += torch.numel(loss_mask.data)
total_traj += pred_traj_gt.size(1)
if total_traj >= NUM_SAMPLE_CHECK:
break
metrics['d_loss'] = sum(d_losses) / len(d_losses)
metrics['g_l2_loss_abs'] = sum(g_l2_losses_abs) / loss_mask_sum
metrics['g_l2_loss_rel'] = sum(g_l2_losses_rel) / loss_mask_sum
metrics['ade'] = sum(disp_error) / (total_traj * PRED_LEN)
metrics['fde'] = sum(f_disp_error) / total_traj
metrics['msae'] = sum(mean_speed_disp_error) / (total_traj * PRED_LEN)
metrics['fse'] = sum(final_speed_disp_error) / total_traj
generator.train()
return metrics
def evaluate(loader, generator, num_samples, speed_regressor):
ade_outer, fde_outer, simulated_output, total_traj, sequences, labels, observed_traj = [], [], [], [], [], [], []
with torch.no_grad():
for batch in loader:
if USE_GPU:
batch = [tensor.cuda() for tensor in batch]
else:
batch = [tensor for tensor in batch]
if MULTI_CONDITIONAL_MODEL:
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel,
loss_mask, seq_start_end, obs_ped_speed, pred_ped_speed,
obs_label, pred_label, obs_obj_rel_speed) = batch
else:
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel,
loss_mask, seq_start_end, obs_ped_speed, pred_ped_speed,
obs_obj_rel_speed) = batch
ade, fde, traj_op, traj_obs = [], [], [], []
total_traj.append(pred_traj_gt.size(1))
sequences.append(seq_start_end)
if MULTI_CONDITIONAL_MODEL:
labels.append(torch.cat([obs_label, pred_label], dim=0))
for _ in range(num_samples):
if TEST_METRIC == 1: # USED DURING PREDICTION ENVIRONMENT
if MULTI_CONDITIONAL_MODEL:
_, final_enc_h = generator(obs_traj,
obs_traj_rel,
seq_start_end,
obs_ped_speed,
pred_ped_speed,
pred_traj_gt,
0,
None,
obs_obj_rel_speed,
obs_label=obs_label,
pred_label=pred_label)
fake_speed = speed_regressor(obs_ped_speed,
final_enc_h)
pred_traj_fake_rel, _ = generator(
obs_traj,
obs_traj_rel,
seq_start_end,
obs_ped_speed,
pred_ped_speed,
pred_traj_gt,
TEST_METRIC,
fake_speed,
obs_obj_rel_speed,
obs_label=obs_label,
pred_label=pred_label)
else:
_, final_enc_h = generator(obs_traj,
obs_traj_rel,
seq_start_end,
obs_ped_speed,
pred_ped_speed,
pred_traj_gt,
0,
None,
obs_obj_rel_speed,
obs_label=None,
pred_label=None)
fake_speed = speed_regressor(obs_ped_speed,
final_enc_h)
pred_traj_fake_rel, _ = generator(obs_traj,
obs_traj_rel,
seq_start_end,
obs_ped_speed,
pred_ped_speed,
pred_traj_gt,
TEST_METRIC,
fake_speed,
obs_obj_rel_speed,
obs_label=None,
pred_label=None)
elif TEST_METRIC == 2: # Used during Simulation environment
if MULTI_CONDITIONAL_MODEL:
pred_traj_fake_rel, _ = generator(
obs_traj,
obs_traj_rel,
seq_start_end,
obs_ped_speed,
pred_ped_speed,
pred_traj_gt,
TEST_METRIC,
None,
obs_obj_rel_speed,
obs_label=obs_label,
pred_label=pred_label)
else:
pred_traj_fake_rel, _ = generator(obs_traj,
obs_traj_rel,
seq_start_end,
obs_ped_speed,
pred_ped_speed,
pred_traj_gt,
TEST_METRIC,
None,
obs_obj_rel_speed,
obs_label=None,
pred_label=None)
pred_traj_fake = relative_to_abs(pred_traj_fake_rel,
obs_traj[-1])
ade.append(
displacement_error(pred_traj_fake,
pred_traj_gt,
mode='raw'))
fde.append(
final_displacement_error(pred_traj_fake[-1],
pred_traj_gt[-1],
mode='raw'))
traj_op.append(pred_traj_fake.unsqueeze(dim=0))
traj_obs.append(obs_traj.unsqueeze(dim=0))
best_traj, min_ade_error = evaluate_helper(
torch.stack(ade, dim=1), torch.cat(traj_op, dim=0),
seq_start_end)
staked_obs = torch.cat(traj_obs, dim=0)
obs = staked_obs[0]
observed_traj.append(obs)
_, min_fde_error = evaluate_helper(torch.stack(fde, dim=1),
torch.cat(traj_op, dim=0),
seq_start_end)
ade_outer.append(min_ade_error)
fde_outer.append(min_fde_error)
simulated_output.append(best_traj)
ade = sum(ade_outer) / (sum(total_traj) * PRED_LEN)
fde = sum(fde_outer) / (sum(total_traj))
simulated_traj = torch.cat(simulated_output, dim=1)
total_obs = torch.cat(observed_traj, dim=1).permute(1, 0, 2)
if MULTI_CONDITIONAL_MODEL:
all_labels = torch.cat(labels, dim=1)
last_items_in_sequences = []
curr_sequences = []
i = 0
for sequence_list in sequences:
last_sequence = sequence_list[-1]
if i > 0:
last_items_sum = sum(last_items_in_sequences)
curr_sequences.append(last_items_sum + sequence_list)
last_items_in_sequences.append(last_sequence[1])
if i == 0:
curr_sequences.append(sequence_list)
i += 1
continue
sequences = torch.cat(curr_sequences, dim=0)
colpercent = collisionPercentage(simulated_traj, sequences)
print('Collision Percentage: ', colpercent * 100)
# The user defined speed is verified by computing inverse sigmoid function on the output speed of the model.
if TEST_METRIC == 2:
if SINGLE_CONDITIONAL_MODEL:
verify_speed(simulated_traj, sequences, labels=None)
else:
verify_speed(simulated_traj, sequences, labels=all_labels)
return ade, fde, colpercent * 100
def generator_step(batch, generator, discriminator, g_loss_fn, optimizer_g):
if USE_GPU:
batch = [tensor.cuda() for tensor in batch]
else:
batch = [tensor for tensor in batch]
if MULTI_CONDITIONAL_MODEL:
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, loss_mask, seq_start_end, obs_ped_speed, pred_ped_speed,
obs_label, pred_label, obs_obj_rel_speed) = batch
else:
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, loss_mask, seq_start_end, obs_ped_speed, pred_ped_speed, obs_obj_rel_speed) = batch
losses = {}
loss = torch.zeros(1).to(pred_traj_gt)
g_l2_loss_rel = []
loss_mask = loss_mask[:, OBS_LEN:]
for _ in range(BEST_K):
if MULTI_CONDITIONAL_MODEL:
generator_out, final_enc_h = generator(obs_traj, obs_traj_rel, seq_start_end, obs_ped_speed, pred_ped_speed,
pred_traj_gt, TRAIN_METRIC, None, obs_obj_rel_speed, obs_label=obs_label, pred_label=pred_label)
else:
generator_out, final_enc_h = generator(obs_traj, obs_traj_rel, seq_start_end, obs_ped_speed, pred_ped_speed,
pred_traj_gt, TRAIN_METRIC, None, obs_obj_rel_speed, obs_label=None, pred_label=None)
pred_traj_fake_rel = generator_out
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
if L2_LOSS_WEIGHT > 0:
g_l2_loss_rel.append(L2_LOSS_WEIGHT * l2_loss(
pred_traj_fake_rel,
pred_traj_gt_rel,
loss_mask,
mode='raw'))
g_l2_loss_sum_rel = torch.zeros(1).to(pred_traj_gt)
if L2_LOSS_WEIGHT > 0:
g_l2_loss_rel = torch.stack(g_l2_loss_rel, dim=1)
for start, end in seq_start_end.data:
_g_l2_loss_rel = g_l2_loss_rel[start:end]
_g_l2_loss_rel = torch.sum(_g_l2_loss_rel, dim=0)
_g_l2_loss_rel = torch.min(_g_l2_loss_rel) / torch.sum(loss_mask[start:end])
g_l2_loss_sum_rel += _g_l2_loss_rel
losses['G_l2_loss_rel'] = g_l2_loss_sum_rel.item()
loss += g_l2_loss_sum_rel
traj_fake = torch.cat([obs_traj, pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel, pred_traj_fake_rel], dim=0)
ped_speed = torch.cat([obs_ped_speed, pred_ped_speed], dim=0)
if MULTI_CONDITIONAL_MODEL:
label_info = torch.cat([obs_label, pred_label], dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel, ped_speed, label=label_info)
else:
scores_fake = discriminator(traj_fake, traj_fake_rel, ped_speed, label=None)
discriminator_loss = g_loss_fn(scores_fake)
loss += discriminator_loss
losses['G_discriminator_loss'] = discriminator_loss.item()
losses['G_total_loss'] = loss.item()
optimizer_g.zero_grad()
loss.backward()
optimizer_g.step()
return losses
def plot_trajectory(args, loader, generator):
ground_truth_input = []
all_model_output_traj = []
ground_truth_output = []
pic_cnt = 0
with torch.no_grad():
for batch in loader:
batch = [tensor.cuda() for tensor in batch]
(
obs_traj,
pred_traj_gt,
obs_traj_rel,
pred_traj_gt_rel,
non_linear_ped,
loss_mask,
seq_start_end,
) = batch
ade = []
ground_truth_input.append(obs_traj)
ground_truth_output.append(pred_traj_gt)
model_output_traj = []
model_output_traj_best = torch.ones_like(pred_traj_gt).cuda()
for _ in range(args.num_samples):
pred_traj_fake_rel = generator(
obs_traj_rel, obs_traj, seq_start_end, 0, 3
)
pred_traj_fake_rel = pred_traj_fake_rel[-args.pred_len :]
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
model_output_traj.append(pred_traj_fake)
ade_, fde_ = cal_ade_fde(pred_traj_gt, pred_traj_fake)
ade.append(ade_)
model_output_traj_best = evaluate_helper(
ade, seq_start_end, model_output_traj, model_output_traj_best
)
all_model_output_traj.append(model_output_traj_best)
for (start, end) in seq_start_end:
plt.figure(figsize=(20,15), dpi=100)
ground_truth_input_x_piccoor = (
obs_traj[:, start:end, :].cpu().numpy()[:, :, 0].T
)
ground_truth_input_y_piccoor = (
obs_traj[:, start:end, :].cpu().numpy()[:, :, 1].T
)
ground_truth_output_x_piccoor = (
pred_traj_gt[:, start:end, :].cpu().numpy()[:, :, 0].T
)
ground_truth_output_y_piccoor = (
pred_traj_gt[:, start:end, :].cpu().numpy()[:, :, 1].T
)
model_output_x_piccoor = (
model_output_traj_best[:, start:end, :].cpu().numpy()[:, :, 0].T
)
model_output_y_piccoor = (
model_output_traj_best[:, start:end, :].cpu().numpy()[:, :, 1].T
)
for i in range(ground_truth_output_x_piccoor.shape[0]):
observed_line = plt.plot(
ground_truth_input_x_piccoor[i, :],
ground_truth_input_y_piccoor[i, :],
"r-",
linewidth=4,
label="Observed Trajectory",
)[0]
observed_line.axes.annotate(
"",
xytext=(
ground_truth_input_x_piccoor[i, -2],
ground_truth_input_y_piccoor[i, -2],
),
xy=(
ground_truth_input_x_piccoor[i, -1],
ground_truth_input_y_piccoor[i, -1],
),
arrowprops=dict(
arrowstyle="->", color=observed_line.get_color(), lw=1
),
size=20,
)
ground_line = plt.plot(
np.append(
ground_truth_input_x_piccoor[i, -1],
ground_truth_output_x_piccoor[i, :],
),
np.append(
ground_truth_input_y_piccoor[i, -1],
ground_truth_output_y_piccoor[i, :],
),
"b-",
linewidth=4,
label="Ground Truth",
)[0]
predict_line = plt.plot(
np.append(
ground_truth_input_x_piccoor[i, -1],
model_output_x_piccoor[i, :],
),
np.append(
ground_truth_input_y_piccoor[i, -1],
model_output_y_piccoor[i, :],
),
color="#ffff00",
ls="--",
linewidth=4,
label="Predicted Trajectory",
)[0]
#plt.axis("off")
plt.savefig(
"./traj_fig/pic_{}.png".format(pic_cnt)
)
plt.close()
pic_cnt += 1
def check_accuracy(loader, generator, discriminator, d_loss_fn, speed_regressor):
d_losses = []
metrics = {}
g_l2_losses_abs, g_l2_losses_rel = ([],) * 2
disp_error, f_disp_error, mean_speed_disp_error, final_speed_disp_error = [], [], [], []
total_traj = 0
loss_mask_sum = 0
generator.eval()
with torch.no_grad():
for batch in loader:
if USE_GPU:
batch = [tensor.cuda() for tensor in batch]
else:
batch = [tensor for tensor in batch]
if MULTI_CONDITIONAL_MODEL:
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, loss_mask, seq_start_end, obs_ped_speed,
pred_ped_speed, obs_label, pred_label, obs_obj_rel_speed) = batch
else:
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, loss_mask, seq_start_end, obs_ped_speed,
pred_ped_speed, obs_obj_rel_speed) = batch
if MULTI_CONDITIONAL_MODEL:
pred_traj_fake_rel, final_enc_h = generator(obs_traj, obs_traj_rel, seq_start_end, obs_ped_speed, pred_ped_speed,
pred_traj_gt, TRAIN_METRIC, None, obs_obj_rel_speed, obs_label=obs_label, pred_label=pred_label)
else:
pred_traj_fake_rel, final_enc_h = generator(obs_traj, obs_traj_rel, seq_start_end, obs_ped_speed, pred_ped_speed,
pred_traj_gt, TRAIN_METRIC, None, obs_obj_rel_speed, obs_label=None, pred_label=None)
fake_ped_speed = speed_regressor(obs_ped_speed, final_enc_h)
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
loss_mask = loss_mask[:, OBS_LEN:]
g_l2_loss_abs, g_l2_loss_rel = cal_l2_losses(
pred_traj_gt, pred_traj_gt_rel, pred_traj_fake,
pred_traj_fake_rel, loss_mask
)
abs_speed_los = cal_mae_speed_loss(pred_ped_speed, fake_ped_speed)
ade = displacement_error(pred_traj_gt, pred_traj_fake)
fde = final_displacement_error(pred_traj_gt, pred_traj_fake)
traj_real = torch.cat([obs_traj, pred_traj_gt], dim=0)
traj_real_rel = torch.cat([obs_traj_rel, pred_traj_gt_rel], dim=0)
traj_fake = torch.cat([obs_traj, pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel, pred_traj_fake_rel], dim=0)
ped_speed = torch.cat([obs_ped_speed, pred_ped_speed], dim=0)
if MULTI_CONDITIONAL_MODEL:
label_info = torch.cat([obs_label, pred_label], dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel, ped_speed, label=label_info)
scores_real = discriminator(traj_real, traj_real_rel, ped_speed, label=label_info)
else:
scores_fake = discriminator(traj_fake, traj_fake_rel, ped_speed, label=None)
scores_real = discriminator(traj_real, traj_real_rel, ped_speed, label=None)
d_loss = d_loss_fn(scores_real, scores_fake)
d_losses.append(d_loss.item())
g_l2_losses_abs.append(g_l2_loss_abs.item())
g_l2_losses_rel.append(g_l2_loss_rel.item())
disp_error.append(ade.item())
f_disp_error.append(fde.item())
loss_mask_sum += torch.numel(loss_mask.data)
total_traj += pred_traj_gt.size(1)
if total_traj >= NUM_SAMPLE_CHECK:
break
metrics['d_loss'] = sum(d_losses) / len(d_losses)
metrics['g_l2_loss_abs'] = sum(g_l2_losses_abs) / loss_mask_sum
metrics['g_l2_loss_rel'] = sum(g_l2_losses_rel) / loss_mask_sum
metrics['ade'] = sum(disp_error) / (total_traj * PRED_LEN)
metrics['fde'] = sum(f_disp_error) / total_traj
generator.train()
return metrics
def check_accuracy(args, loader, generator, discriminator, d_loss_fn,
N_VALID_EXAMPLES):
d_losses = []
metrics = {}
g_l2_losses_abs, g_l2_losses_rel = ([], ) * 2
disp_error, disp_error_l, disp_error_nl = ([], ) * 3
f_disp_error, f_disp_error_l, f_disp_error_nl = ([], ) * 3
total_traj, total_traj_l, total_traj_nl = 0, 0, 0
loss_mask_sum = 0
generator.eval()
with torch.no_grad():
for batch_idx, batch in enumerate(loader):
if args.use_gpu == 1:
batch = [tensor.cuda() for tensor in batch]
else:
batch = [tensor for tensor in batch]
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel,
non_linear_ped, loss_mask, seq_start_end) = batch
linear_ped = 1 - non_linear_ped
loss_mask = loss_mask[:, args.obs_len:]
pred_traj_fake_rel = generator(obs_traj, obs_traj_rel,
seq_start_end)
pred_traj_fake = relative_to_abs(pred_traj_fake_rel, obs_traj[-1])
g_l2_loss_abs, g_l2_loss_rel = cal_l2_losses(
pred_traj_gt, pred_traj_gt_rel, pred_traj_fake,
pred_traj_fake_rel, loss_mask)
ade, ade_l, ade_nl = cal_ade(pred_traj_gt, pred_traj_fake,
linear_ped, non_linear_ped)
fde, fde_l, fde_nl = cal_fde(pred_traj_gt, pred_traj_fake,
linear_ped, non_linear_ped)
traj_real = torch.cat([obs_traj, pred_traj_gt], dim=0)
traj_real_rel = torch.cat([obs_traj_rel, pred_traj_gt_rel], dim=0)
traj_fake = torch.cat([obs_traj, pred_traj_fake], dim=0)
traj_fake_rel = torch.cat([obs_traj_rel, pred_traj_fake_rel],
dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel,
seq_start_end)
scores_real = discriminator(traj_real, traj_real_rel,
seq_start_end)
d_loss = d_loss_fn(scores_real, scores_fake)
d_losses.append(d_loss.item())
g_l2_losses_abs.append(g_l2_loss_abs.item())
g_l2_losses_rel.append(g_l2_loss_rel.item())
disp_error.append(ade.item())
disp_error_l.append(ade_l.item())
disp_error_nl.append(ade_nl.item())
f_disp_error.append(fde.item())
f_disp_error_l.append(fde_l.item())
f_disp_error_nl.append(fde_nl.item())
loss_mask_sum += torch.numel(loss_mask.data)
total_traj += pred_traj_gt.size(1)
total_traj_l += torch.sum(linear_ped).item()
total_traj_nl += torch.sum(non_linear_ped).item()
if batch_idx * args.batch_size >= N_VALID_EXAMPLES:
break
metrics['d_loss'] = sum(d_losses) / len(d_losses)
metrics['g_l2_loss_abs'] = sum(g_l2_losses_abs) / loss_mask_sum
metrics['g_l2_loss_rel'] = sum(g_l2_losses_rel) / loss_mask_sum
metrics['ade'] = sum(disp_error) / (total_traj * args.pred_len)
metrics['fde'] = sum(f_disp_error) / total_traj
generator.train()
return metrics
def check_accuracy(loader, generator, discriminator, d_loss_fn, limit=False):
d_losses = [] #
metrics = {}
g_l2_losses_abs, g_l2_losses_rel = ([], ) * 2
disp_error = [] # ADE FDE
f_disp_error = []
total_traj = 0
mask_sum = 0
generator.eval()
with torch.no_grad(): #
for batch in loader:
batch = [tensor.cuda() for tensor in batch]
# (obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, vgg_list) = batch
# (obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel) = batch
(obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, n_l, l_m,
V_obs, A_obs, V_pre, A_pre, vgg_list) = batch
# pred_traj_fake_rel = generator(obs_traj, obs_traj_rel, vgg_list)
pred_traj_fake_rel = generator(obs_traj, obs_traj_rel, V_obs,
A_obs, vgg_list) # T V C
pred_traj_fake = relative_to_abs(pred_traj_fake_rel,
obs_traj[0, :, :,
-1]) # T V C——V C
g_l2_loss_abs = l2_loss(pred_traj_fake, pred_traj_gt, mode='sum')
g_l2_loss_rel = l2_loss(pred_traj_fake_rel,
pred_traj_gt_rel,
mode='sum')
ade = displacement_error(pred_traj_fake, pred_traj_gt) # TVC NVCT
fde = final_displacement_error(pred_traj_fake[-1],
pred_traj_gt[0, :, :, -1]) # VC VC
traj_real = torch.cat([obs_traj[:, :, 0, :], pred_traj_gt], dim=0)
traj_real_rel = torch.cat(
[obs_traj_rel[:, :, 0, :], pred_traj_gt_rel], dim=0)
traj_fake = torch.cat([obs_traj[:, :, 0, :], pred_traj_fake],
dim=0)
traj_fake_rel = torch.cat(
[obs_traj_rel[:, :, 0, :], pred_traj_fake_rel], dim=0)
scores_fake = discriminator(traj_fake, traj_fake_rel)
scores_real = discriminator(traj_real, traj_real_rel)
d_loss = d_loss_fn(scores_real, scores_fake)
d_losses.append(d_loss.item())
g_l2_losses_abs.append(g_l2_loss_abs.item())
g_l2_losses_rel.append(g_l2_loss_rel.item())
disp_error.append(ade.item())
f_disp_error.append(fde.item())
mask_sum += (pred_traj_gt.size(1) * PRED_LEN)
total_traj += pred_traj_gt.size(1)
if limit and total_traj >= NUM_SAMPLES_CHECK:
break
metrics['d_loss'] = sum(d_losses) / len(d_losses)
metrics['g_l2_loss_abs'] = sum(g_l2_losses_abs) / mask_sum
metrics['g_l2_loss_rel'] = sum(g_l2_losses_rel) / mask_sum
metrics['ade'] = sum(disp_error) / (total_traj * PRED_LEN)
metrics['fde'] = sum(f_disp_error) / total_traj
generator.train()
return metrics
