def get_mask_ts(mask, target):
mask = nn.Sigmoid()(mask) > 0.5
temp_tensor = torch.zeros(1, 3, 400, 400)
temp_target = [{'masks': mask, 'boxes': torch.tensor([[1., 1., 1., 1.]])}]
_, temp_target = target_transform(temp_tensor, temp_target)
predicted_road_map = temp_target[0]['masks'][0, :1]
ts_road_map = compute_ts_road_map(predicted_road_map, target[0]['masks'])
return ts_road_map
def train_model(model, optimizer, scheduler, num_epochs=25):
best_ts = 0.0
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
since = time.time()
for phase in ['train', 'val']:
if phase == 'train':
scheduler.step()
model.train()
else:
model.eval()
threat_score = 0.0
loss_tot = 0.0
for i, (samples, targets, road_images,
extras) in enumerate(dataloaders[phase]):
inputs = torch.stack(samples).to(device)
labels = torch.stack(road_images).to(device)
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs, mu, logvar = model(inputs)
loss = loss_function(outputs, labels, mu, logvar)
if phase == 'train':
loss.backward()
optimizer.step()
if phase == 'val':
for output, label in zip(outputs, labels):
output = output.detach().cpu().numpy()
label = label.detach().cpu().numpy()
ts_road_map = compute_ts_road_map(
output > 0.5, label)
threat_score += ts_road_map
loss_tot += loss.item()
if phase == 'train':
print('train loss: {}'.format(loss_tot /
len(labeled_trainset)))
elif phase == 'val':
print('val loss: {}'.format(loss_tot / len(labeled_valset)))
epoch_ts = threat_score.item() / len(labeled_valset)
print('ts score: {}'.format(epoch_ts))
if epoch_ts > best_ts:
print("saving best model at epoch {}".format(epoch))
best_ts = epoch_ts
torch.save(model.state_dict(),
"models_pkl/ResNetVAE_0507_main.pkl")
# torch.save(optimizer.state_dict(), "models_pkl/ResNetVAE_optimizer_0507.pkl")
# torch.save(scheduler.state_dict(), "models_pkl/ResNetVAE_scheduler_0507.pkl")
time_elapsed = time.time() - since
print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
def report_loss(self, split_name="val", model_path=None, display_images=True,
batches_idces_to_display=slice(None), threshold = 0.5):
assert split_name in ["train", "val", "test"]
print('Estimating performance on {} set'.format(split_name))
if model_path is not None:
print('Restoring Best checkpoint from {}'.format(model_path))
state = torch.load(model_path)
# self.restore_model()
self.model.load_state_dict(state['state_dict'])
self.model.eval()
threat_score = 0.0
total = 0.0
batches_idces_to_display = np.arange(len(self.dataloaders[split_name]))[batches_idces_to_display]
for i, temp_batch in enumerate(self.dataloaders[split_name]):
if i not in batches_idces_to_display:
continue
total += len(temp_batch[0])
samples, targets, road_images, extra = temp_batch
samples = torch.stack(samples).to(self.device)
road_images = torch.stack(road_images).to(self.device)
if self.model_type == 'ae':
pred_maps = self.model(samples)
elif self.model_type == 'vae':
pred_maps, mu, logvar = self.model(samples,is_training=False)
for pred_map, road_image in zip(pred_maps,road_images):
ts_road_map = compute_ts_road_map(pred_map > threshold, road_image)
threat_score += ts_road_map
if display_images:
for sample, pred_map, road_image in zip(samples,pred_maps,road_images):
print('Test Batch: {}'.format(i))
self.show_predictions(sample, pred_map, road_image, threshold)
# CAM_FRONT_LEFT, CAM_FRONT, CAM_FRONT_RIGHT, CAM_BACK_LEFT, CAM_BACK, CAM_BACK_RIGHT
threat_score /= total
print('Total samples: {}, Total Threat Score: {}'.format(total,total*threat_score))
print('Mean Threat Score is: {}'.format(threat_score))
total_ts_road_map = 0
with torch.no_grad():
for i, data in enumerate(dataloader_task1):
total += 1
sample, target, road_image = data
sample = sample.cuda()
predicted_bounding_boxes = model_loader.get_bounding_boxes(
sample)[0].cpu()
ats_bounding_boxes, iou_max = compute_ats_bounding_boxes(
predicted_bounding_boxes, target['bounding_box'][0])
total_ats_bounding_boxes += ats_bounding_boxes
if opt.verbose:
print(f'{i} - Bounding Box Score: {ats_bounding_boxes:.4}')
for i, data in enumerate(dataloader_task2):
sample, target, road_image = data
sample = sample.cuda()
predicted_road_map = model_loader.get_binary_road_map(sample).cpu()
ts_road_map = compute_ts_road_map(predicted_road_map, road_image)
total_ts_road_map += ts_road_map
if opt.verbose:
print(f'{i} - Road Map Score: {ts_road_map:.4}')
print(
f'{model_loader.team_name} - {model_loader.round_number} - Bounding Box Score: {total_ats_bounding_boxes / total:.4} - Road Map Score: {total_ts_road_map / total:.4}'
)
def train(self, num_epochs, threshold, tensorboard_log_dir):
while self.epoch < num_epochs:
print('Epoch {}/{}'.format(self.epoch, num_epochs - 1))
print('-' * 10)
self.epoch += 1
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
self.model.train() # Set model to training mode
else:
self.model.eval() # Set model to evaluate mode
running_loss = 0.0
threat_score = 0.0
# Iterate over data.
for i, temp_batch in enumerate(tqdm(self.dataloaders[phase])):
samples, targets, road_images, extras = temp_batch
samples = torch.stack(samples).to(self.device)
road_images = torch.stack(road_images).to(self.device)
# zero the parameter gradients
self.optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
if self.model_type == 'ae':
pred_maps = self.model(samples)
loss = self.criterion(pred_maps, road_images.float())
elif self.model_type == 'vae':
pred_maps, mu, logvar = self.model(samples,phase == 'train')
loss, CE, KLD = loss_function(pred_maps, road_images, mu, logvar)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
self.optimizer.step()
self.scheduler.step()
else:
for pred_map,road_image in zip(pred_maps,road_images):
ts_road_map = compute_ts_road_map(pred_map > threshold, road_image)
threat_score += ts_road_map
running_loss += loss.item() #*batch_size
# tensorboard logging
if phase == 'train':
writer_idx = self.epoch * len(self.train_loader) + i
self.writer.add_scalar(phase+'_loss', loss.item(), writer_idx)
if self.model_type == 'vae':
self.writer.add_scalar(phase+'_loss_CE', CE.item(), writer_idx)
self.writer.add_scalar(phase+'_loss_KLD', KLD.item(), writer_idx)
pass # end for batch loop
# statistics
if phase == 'train':
running_loss = running_loss / len(self.train_loader.dataset) # per batch, per sample
print(phase, 'running_loss:', round(running_loss,4))
self.writer.add_scalar(phase+'_total_loss', running_loss, self.epoch)
else:
num_test_samples = len(self.val_loader.dataset)
running_loss = running_loss / num_test_samples
print(phase,'running_loss:', round(running_loss,4), 'cumulative_threat_score:', round(threat_score.item(),4), \
'val_len:', num_test_samples, '\n mean_threat_score:',round(threat_score.item() / num_test_samples,4))#, iou / len(val_set))
self.writer.add_scalar(phase+'_ts', threat_score.item()/num_test_samples, self.epoch)
self.writer.add_scalar(phase+'_total_loss', running_loss, self.epoch)
self._update_states(running_loss, threat_score)
def evaluate(self, debug=False):
"""
Evaluation model on task.
---------------------
Returns:
Average validation loss
"""
# puts model in evaluation mode
self.model.eval()
avg_road_ts = 0
avg_box_ats = 0
pred_boxes = []
pred_roads = []
road_ts = []
box_ats = []
# stop gradient tracking
with torch.no_grad():
for i, batch in enumerate(
tqdm(self.val_dataloader,
desc="Validation",
mininterval=30)):
inputs = self.pack_input(batch)
# {"images": batch[0],
# "box_targets": batch[1],
# "road_targets": batch[2]}
out = self.model(**inputs)
# out[0] First element as loss
# out[1] Second element as predicted bounding boxes( for evaluation.o.w.empty list)
# out[2] Third element as road map
# out[3] road_loss
# out[4] box_loss(for train.o.w. 0)
pred_boxes = out[1]
pred_roads = out[2]
for road_map1, road_map2 in zip(pred_roads,
inputs['road_targets']):
road_ts.append(
helper.compute_ts_road_map(road_map1,
road_map2).item())
for boxes1, boxes2 in zip(pred_boxes, inputs['box_targets']):
box_ats.append(
helper.compute_ats_bounding_boxes(
boxes1, boxes2['bounding_box']).item())
if i == 0 and debug:
log.info('Debug')
break
if debug:
log.info(f"Predicted roads is {pred_roads}")
log.info(f"True roads is {inputs['road_targets']}")
log.info(f"Road ts array is {road_ts}")
log.info(f"Predicted roads is {pred_boxes}")
log.info(f"True roads is {inputs['box_targets']}")
log.info(f"Box ats array is {box_ats}")
if len(road_ts) > 0:
avg_road_ts = mean(road_ts)
if len(box_ats) > 0:
avg_box_ats = mean(box_ats)
return avg_road_ts, avg_box_ats
total += 1
sample, target, road_image = data
sample = sample.cuda()
predicted_bounding_boxes = model_loader.get_bounding_boxes(
sample)[0].cpu()
ats_bounding_boxes = compute_ats_bounding_boxes(
predicted_bounding_boxes, target['bounding_box'][0])
total_ats_bounding_boxes += ats_bounding_boxes
if opt.verbose:
print(f'{i} - Bounding Box Score: {ats_bounding_boxes:.4}')
for i, data in enumerate(dataloader_task2):
sample, target, road_image = data
sample = sample.cuda()
predicted_road_map = model_loader.get_binary_road_map(sample).cpu()
ts_road_map = compute_ts_road_map(predicted_road_map.float(),
road_image.float())
total_ts_road_map += ts_road_map
if opt.verbose:
print(f'{i} - Road Map Score: {ts_road_map:.4}')
# print(f'{model_loader.team_name} - {model_loader.round_number} - Bounding Box Score: {total_ats_bounding_boxes / total:.4} - Road Map Score: {total_ts_road_map / total:.4}')
print(
f'{model_loader.team_name} - {model_loader.round_number} - Road Map Score: {total_ts_road_map / total:.4}'
)
def validation(self):
if self.opt.type == "both":
iou_static, iou_dynamic, mAP_static, mAP_dynamic = np.array([
0., 0.
]), np.array([0., 0.]), np.array([0., 0.]), np.array([0., 0.])
threat_static, threat_dynamic = 0, 0
total_images = 0
for batch_idx, ipts in tqdm.tqdm(enumerate(self.val_loader)):
with torch.no_grad():
inputs, outputs = self.process_batch(ipts, True)
pred_static = torch.argmax(outputs["static"].detach(), 1).cpu(
) #.numpy() #converting to cpu top copy tensor to host memory - Trevor
pred_dynamic = torch.argmax(outputs["dynamic"].detach(),
1).cpu() #.numpy() #same Trevor
true_static = torch.squeeze(
inputs["static"],
1).detach().cpu() #.numpy() #same Trevor
true_dynamic = torch.squeeze(
inputs["dynamic"],
1).detach().cpu() #.numpy() #same Trevor
print("pred shape", pred_static.shape, "true shape",
true_static.shape, pred_static.dtype, true_static.dtype)
threat_static += compute_ts_road_map(pred_static.float(),
true_static.float())
threat_dynamic += compute_ts_road_map(pred_dynamic.float(),
true_dynamic.float())
for bb in range(pred_static.shape[0]):
total_images += 1
iou_static += mean_IU(pred_static[bb].numpy(),
true_static[bb].numpy())
iou_dynamic += mean_IU(pred_dynamic[bb].numpy(),
true_dynamic[bb].numpy())
mAP_static += mean_precision(pred_static[bb].numpy(),
true_static[bb].numpy())
mAP_dynamic += mean_precision(pred_dynamic[bb].numpy(),
true_dynamic[bb].numpy())
print("total images", total_images)
print(len(self.val_loader))
iou_static /= total_images #len(self.val_loader)
mAP_static /= total_images #len(self.val_loader)
threat_static /= total_images #len(self.val_loader)
iou_dynamic /= total_images #len(self.val_loader)
mAP_dynamic /= total_images #len(self.val_loader)
threat_dynamic /= total_images #len(self.val_loader)
# print("Epoch: %d | Validation: Static: mIOU: %.8f mAP: %.4f Dynamic: mIOU: %.8f mAP: %.4f"%(self.epoch, iou_static[1], mAP_static[1], iou_dynamic[1], mAP_dynamic[1]))
print(
"Epoch: %d | Validation: Static: mTS: %.8f mAP: %.4f Dynamic: mTS: %.8f mAP: %.4f iou_static %.8f iou_dynamic %.8f"
% (self.epoch, threat_static, mAP_static[1], threat_dynamic,
mAP_dynamic[1], iou_static[1], iou_dynamic[1]))
return threat_static + threat_dynamic #may want to see about having two returns to save independently
else:
iou, mAP = np.array([0., 0.]), np.array([0., 0.])
threat = 0
for batch_idx, ipts in tqdm.tqdm(enumerate(self.val_loader)):
with torch.no_grad():
inputs, outputs = self.process_batch(ipts, True)
pred = torch.argmax(outputs["topview"].detach(),
1).cuda().numpy()
true = torch.squeeze(inputs[self.opt.type],
1).detach().cuda().numpy()
#print(pred.shape, true.shape)
threat += compute_ts_road_map(pred, true)
for bb in range(pred.shape[0]):
# iou += mean_IU(pred[bb], true[bb])
mAP += mean_precision(pred[bb], true[bb])
iou /= len(self.val_loader)
mAP /= len(self.val_loader)
threat /= len(self.val_loader)
# print("Epoch: %d | Validation: mIOU: %.4f mAP: %.4f"%(self.epoch, iou[1], mAP[1]))
print("Epoch: %d | Validation: mTS: %.4f mAP: %.4f" %
(self.epoch, threat, mAP[1]))
return threat
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, i * len(sample), len(trainloader.dataset),
100. * i / len(trainloader), loss.item()))
print("\n Average Train Epoch Loss for epoch {} is {} ", epoch+1, np.mean(train_losses))
#### validation logic ####
model.eval()
val_losses = []
threat_score = []
for i, (sample, target, road_img) in enumerate(valloader):
sample = torch.stack(sample).to(device)
batch_size = sample.shape[0]
sample = sample.view(batch_size, -1, 256, 306) # size: ([3, 18, 256, 306])
road_img = torch.stack(road_img).float().to(device)
with torch.no_grad():
pred_map = model(sample)
loss = criterion(pred_map, road_img)
val_losses.append(loss.item())
out_map = (pred_map > 0.5).float()
threat_score.append(compute_ts_road_map(out_map, road_img).item())
print("Validation Epoch: {}, Average Validation Epoch Loss: {}".format(epoch, np.mean(val_losses)))
print("Average Threat Score: {} ".format(np.mean(threat_score)))
if np.mean(val_losses) < best_val_loss:
best_val_loss = np.mean(val_losses)
torch.save(model.state_dict(), 'test.pt')
def validation(epoch, batch_i, batch, param):
with torch.no_grad():
outputs = None
if not param['run_name'] in ['bbox_reg', 'bbox_reg_full']:
inputs, labels = batch
inputs = inputs.to(param['device'])
labels = labels.to(param['device'])
else:
inputs, samples, labels, graph = batch
inputs = inputs.to(param['device'])
labels = labels.to(param['device'])
samples = samples.to(param['device'])
if param['run_name'] == 'mosaic':
fusion_layer = param['model'][0].eval()
static_model = param['model'][1].eval()
fusion_outputs = fusion_layer(inputs)
outputs = static_model(fusion_outputs).squeeze(1)
elif param['run_name'] == 'polar':
static_polar = param['model'].eval()
outputs = static_polar(inputs.squeeze(1)).squeeze(1)
elif param['run_name'] in ['front', 'bbox']:
static_front = param['model'].eval()
outputs = static_front(inputs).squeeze(1)
for i in range(outputs.shape[0]):
ts = compute_ts_road_map(outputs[i], labels[i])
param['ts'] += ts
elif param['run_name'] in ['camerabased']:
inputs = inputs.view(-1, 256, 16, 20)
labels = labels.view(-1, 400, 538)
static_camerabased = param['model'].eval()
outputs = static_camerabased(inputs).squeeze(1)
elif param['run_name'] == 'bbox_reg':
camera_model = param['model'][0].eval()
bbox_model = param['model'][1].eval()
classifier = param['model'][2].eval()
camera_feature = camera_model(inputs).unsqueeze(1)
bbox_feature = bbox_model(samples.view(-1, 8)) # Batched bbox
camera_feature_batch = camera_feature.repeat(
1, samples.shape[1],
1).view(-1, 1024) # repeat to match num of bbox features
bbox_feature_positive = bbox_feature[labels.reshape(-1) == 1]
camera_feature_batch_positive = camera_feature_batch[
labels.reshape(-1) == 1]
labels = labels.reshape(-1)[labels.reshape(-1) == 1]
outputs = classifier(camera_feature_batch_positive,
bbox_feature_positive).squeeze(1)
gen_bbox_heatmap(param, graph, camera_feature, epoch, batch_i)
elif param['run_name'] == 'camerabased_full':
inputs = inputs.view(-1, 256, 16, 20)
labels = labels.view(-1, 800, 800)
static_camerabased = param['model'][0]
fusion_cameras = param['model'][1].eval()
# [batch*6, 400, 538]
outputs_cameras = static_camerabased(inputs).squeeze(1)
# [batch, 6, 400, 538]
outputs_cameras = outputs_cameras.view(-1, 6, 400, 538)
# [batch, 800, 800]
outputs = fusion_cameras(outputs_cameras).squeeze(1)
elif param['run_name'] == 'camerabased_unet':
inputs = inputs.view(-1, 256, 16, 20)
labels = labels.view(-1, 800, 800)
static_camerabased = param['model'][0]
unet_cameras = param['model'][1].eval()
# [batch*6, 400, 538]
outputs_cameras = static_camerabased(inputs).squeeze(1)
# [batch, 6, 400, 538]
outputs_cameras = outputs_cameras.view(-1, 6, 400, 538)
# [batch, 1, 800, 800]
unet_input = concat_cameras(
outputs_cameras.cpu().numpy()).unsqueeze(1).to(param['device'])
# [batch, 800, 800]
outputs = unet_cameras(unet_input).squeeze(1)
loss = param['criterion'](outputs, labels.float())
param['running_loss'] += loss.item()
if epoch % 5 == 0 and batch_i % 100 == 1 and param[
'run_name'] != 'bbox_reg':
sample_path = 'sample_output_{}'.format(param['run_name'])
if not os.path.exists(sample_path):
os.mkdir(sample_path)
else:
rand_camera = random.randrange(0, outputs.shape[0])
cv2.imwrite(
'{}/val_sample_{}_{}_{}.png'.format(
sample_path, epoch, batch_i, rand_camera),
outputs[rand_camera].detach().cpu().numpy() * 255)
print("\n Average Train Epoch Loss for epoch {} is {} ", epoch + 1,
np.mean(train_losses))
#### validation logic ####
model.eval()
val_losses = []
threat_score = []
for i, (sample, target, road_img) in enumerate(valloader):
sample = torch.stack(sample).to(device)
batch_size = sample.shape[0]
sample = sample.view(batch_size, -1, 256,
306) # size: ([3, 18, 256, 306])
road_img = torch.stack(road_img).float().to(device)
with torch.no_grad():
pred_map = model(sample)
loss = criterion(pred_map, road_img)
val_losses.append(loss.item())
out_map = (pred_map > 0.5).float()
threat_score.append(
compute_ts_road_map(out_map, road_img).item())
print("Validation Epoch: {}, Average Validation Epoch Loss: {}".
format(epoch, np.mean(val_losses)))
print("Average Threat Score: {} ".format(np.mean(threat_score)))
if np.mean(val_losses) < best_val_loss:
best_val_loss = np.mean(val_losses)
torch.save(model.state_dict(), 'test.pt')