def get_mask_ts(mask, target): mask = nn.Sigmoid()(mask) > 0.5 temp_tensor = torch.zeros(1, 3, 400, 400) temp_target = [{'masks': mask, 'boxes': torch.tensor([[1., 1., 1., 1.]])}] _, temp_target = target_transform(temp_tensor, temp_target) predicted_road_map = temp_target[0]['masks'][0, :1] ts_road_map = compute_ts_road_map(predicted_road_map, target[0]['masks']) return ts_road_map
def train_model(model, optimizer, scheduler, num_epochs=25): best_ts = 0.0 for epoch in range(num_epochs): print('Epoch {}/{}'.format(epoch, num_epochs - 1)) print('-' * 10) since = time.time() for phase in ['train', 'val']: if phase == 'train': scheduler.step() model.train() else: model.eval() threat_score = 0.0 loss_tot = 0.0 for i, (samples, targets, road_images, extras) in enumerate(dataloaders[phase]): inputs = torch.stack(samples).to(device) labels = torch.stack(road_images).to(device) optimizer.zero_grad() with torch.set_grad_enabled(phase == 'train'): outputs, mu, logvar = model(inputs) loss = loss_function(outputs, labels, mu, logvar) if phase == 'train': loss.backward() optimizer.step() if phase == 'val': for output, label in zip(outputs, labels): output = output.detach().cpu().numpy() label = label.detach().cpu().numpy() ts_road_map = compute_ts_road_map( output > 0.5, label) threat_score += ts_road_map loss_tot += loss.item() if phase == 'train': print('train loss: {}'.format(loss_tot / len(labeled_trainset))) elif phase == 'val': print('val loss: {}'.format(loss_tot / len(labeled_valset))) epoch_ts = threat_score.item() / len(labeled_valset) print('ts score: {}'.format(epoch_ts)) if epoch_ts > best_ts: print("saving best model at epoch {}".format(epoch)) best_ts = epoch_ts torch.save(model.state_dict(), "models_pkl/ResNetVAE_0507_main.pkl") # torch.save(optimizer.state_dict(), "models_pkl/ResNetVAE_optimizer_0507.pkl") # torch.save(scheduler.state_dict(), "models_pkl/ResNetVAE_scheduler_0507.pkl") time_elapsed = time.time() - since print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
def report_loss(self, split_name="val", model_path=None, display_images=True, batches_idces_to_display=slice(None), threshold = 0.5): assert split_name in ["train", "val", "test"] print('Estimating performance on {} set'.format(split_name)) if model_path is not None: print('Restoring Best checkpoint from {}'.format(model_path)) state = torch.load(model_path) # self.restore_model() self.model.load_state_dict(state['state_dict']) self.model.eval() threat_score = 0.0 total = 0.0 batches_idces_to_display = np.arange(len(self.dataloaders[split_name]))[batches_idces_to_display] for i, temp_batch in enumerate(self.dataloaders[split_name]): if i not in batches_idces_to_display: continue total += len(temp_batch[0]) samples, targets, road_images, extra = temp_batch samples = torch.stack(samples).to(self.device) road_images = torch.stack(road_images).to(self.device) if self.model_type == 'ae': pred_maps = self.model(samples) elif self.model_type == 'vae': pred_maps, mu, logvar = self.model(samples,is_training=False) for pred_map, road_image in zip(pred_maps,road_images): ts_road_map = compute_ts_road_map(pred_map > threshold, road_image) threat_score += ts_road_map if display_images: for sample, pred_map, road_image in zip(samples,pred_maps,road_images): print('Test Batch: {}'.format(i)) self.show_predictions(sample, pred_map, road_image, threshold) # CAM_FRONT_LEFT, CAM_FRONT, CAM_FRONT_RIGHT, CAM_BACK_LEFT, CAM_BACK, CAM_BACK_RIGHT threat_score /= total print('Total samples: {}, Total Threat Score: {}'.format(total,total*threat_score)) print('Mean Threat Score is: {}'.format(threat_score))
total_ts_road_map = 0 with torch.no_grad(): for i, data in enumerate(dataloader_task1): total += 1 sample, target, road_image = data sample = sample.cuda() predicted_bounding_boxes = model_loader.get_bounding_boxes( sample)[0].cpu() ats_bounding_boxes, iou_max = compute_ats_bounding_boxes( predicted_bounding_boxes, target['bounding_box'][0]) total_ats_bounding_boxes += ats_bounding_boxes if opt.verbose: print(f'{i} - Bounding Box Score: {ats_bounding_boxes:.4}') for i, data in enumerate(dataloader_task2): sample, target, road_image = data sample = sample.cuda() predicted_road_map = model_loader.get_binary_road_map(sample).cpu() ts_road_map = compute_ts_road_map(predicted_road_map, road_image) total_ts_road_map += ts_road_map if opt.verbose: print(f'{i} - Road Map Score: {ts_road_map:.4}') print( f'{model_loader.team_name} - {model_loader.round_number} - Bounding Box Score: {total_ats_bounding_boxes / total:.4} - Road Map Score: {total_ts_road_map / total:.4}' )
def train(self, num_epochs, threshold, tensorboard_log_dir): while self.epoch < num_epochs: print('Epoch {}/{}'.format(self.epoch, num_epochs - 1)) print('-' * 10) self.epoch += 1 # Each epoch has a training and validation phase for phase in ['train', 'val']: if phase == 'train': self.model.train() # Set model to training mode else: self.model.eval() # Set model to evaluate mode running_loss = 0.0 threat_score = 0.0 # Iterate over data. for i, temp_batch in enumerate(tqdm(self.dataloaders[phase])): samples, targets, road_images, extras = temp_batch samples = torch.stack(samples).to(self.device) road_images = torch.stack(road_images).to(self.device) # zero the parameter gradients self.optimizer.zero_grad() # forward # track history if only in train with torch.set_grad_enabled(phase == 'train'): if self.model_type == 'ae': pred_maps = self.model(samples) loss = self.criterion(pred_maps, road_images.float()) elif self.model_type == 'vae': pred_maps, mu, logvar = self.model(samples,phase == 'train') loss, CE, KLD = loss_function(pred_maps, road_images, mu, logvar) # backward + optimize only if in training phase if phase == 'train': loss.backward() self.optimizer.step() self.scheduler.step() else: for pred_map,road_image in zip(pred_maps,road_images): ts_road_map = compute_ts_road_map(pred_map > threshold, road_image) threat_score += ts_road_map running_loss += loss.item() #*batch_size # tensorboard logging if phase == 'train': writer_idx = self.epoch * len(self.train_loader) + i self.writer.add_scalar(phase+'_loss', loss.item(), writer_idx) if self.model_type == 'vae': self.writer.add_scalar(phase+'_loss_CE', CE.item(), writer_idx) self.writer.add_scalar(phase+'_loss_KLD', KLD.item(), writer_idx) pass # end for batch loop # statistics if phase == 'train': running_loss = running_loss / len(self.train_loader.dataset) # per batch, per sample print(phase, 'running_loss:', round(running_loss,4)) self.writer.add_scalar(phase+'_total_loss', running_loss, self.epoch) else: num_test_samples = len(self.val_loader.dataset) running_loss = running_loss / num_test_samples print(phase,'running_loss:', round(running_loss,4), 'cumulative_threat_score:', round(threat_score.item(),4), \ 'val_len:', num_test_samples, '\n mean_threat_score:',round(threat_score.item() / num_test_samples,4))#, iou / len(val_set)) self.writer.add_scalar(phase+'_ts', threat_score.item()/num_test_samples, self.epoch) self.writer.add_scalar(phase+'_total_loss', running_loss, self.epoch) self._update_states(running_loss, threat_score)
def evaluate(self, debug=False): """ Evaluation model on task. --------------------- Returns: Average validation loss """ # puts model in evaluation mode self.model.eval() avg_road_ts = 0 avg_box_ats = 0 pred_boxes = [] pred_roads = [] road_ts = [] box_ats = [] # stop gradient tracking with torch.no_grad(): for i, batch in enumerate( tqdm(self.val_dataloader, desc="Validation", mininterval=30)): inputs = self.pack_input(batch) # {"images": batch[0], # "box_targets": batch[1], # "road_targets": batch[2]} out = self.model(**inputs) # out[0] First element as loss # out[1] Second element as predicted bounding boxes( for evaluation.o.w.empty list) # out[2] Third element as road map # out[3] road_loss # out[4] box_loss(for train.o.w. 0) pred_boxes = out[1] pred_roads = out[2] for road_map1, road_map2 in zip(pred_roads, inputs['road_targets']): road_ts.append( helper.compute_ts_road_map(road_map1, road_map2).item()) for boxes1, boxes2 in zip(pred_boxes, inputs['box_targets']): box_ats.append( helper.compute_ats_bounding_boxes( boxes1, boxes2['bounding_box']).item()) if i == 0 and debug: log.info('Debug') break if debug: log.info(f"Predicted roads is {pred_roads}") log.info(f"True roads is {inputs['road_targets']}") log.info(f"Road ts array is {road_ts}") log.info(f"Predicted roads is {pred_boxes}") log.info(f"True roads is {inputs['box_targets']}") log.info(f"Box ats array is {box_ats}") if len(road_ts) > 0: avg_road_ts = mean(road_ts) if len(box_ats) > 0: avg_box_ats = mean(box_ats) return avg_road_ts, avg_box_ats
total += 1 sample, target, road_image = data sample = sample.cuda() predicted_bounding_boxes = model_loader.get_bounding_boxes( sample)[0].cpu() ats_bounding_boxes = compute_ats_bounding_boxes( predicted_bounding_boxes, target['bounding_box'][0]) total_ats_bounding_boxes += ats_bounding_boxes if opt.verbose: print(f'{i} - Bounding Box Score: {ats_bounding_boxes:.4}') for i, data in enumerate(dataloader_task2): sample, target, road_image = data sample = sample.cuda() predicted_road_map = model_loader.get_binary_road_map(sample).cpu() ts_road_map = compute_ts_road_map(predicted_road_map.float(), road_image.float()) total_ts_road_map += ts_road_map if opt.verbose: print(f'{i} - Road Map Score: {ts_road_map:.4}') # print(f'{model_loader.team_name} - {model_loader.round_number} - Bounding Box Score: {total_ats_bounding_boxes / total:.4} - Road Map Score: {total_ts_road_map / total:.4}') print( f'{model_loader.team_name} - {model_loader.round_number} - Road Map Score: {total_ts_road_map / total:.4}' )
def validation(self): if self.opt.type == "both": iou_static, iou_dynamic, mAP_static, mAP_dynamic = np.array([ 0., 0. ]), np.array([0., 0.]), np.array([0., 0.]), np.array([0., 0.]) threat_static, threat_dynamic = 0, 0 total_images = 0 for batch_idx, ipts in tqdm.tqdm(enumerate(self.val_loader)): with torch.no_grad(): inputs, outputs = self.process_batch(ipts, True) pred_static = torch.argmax(outputs["static"].detach(), 1).cpu( ) #.numpy() #converting to cpu top copy tensor to host memory - Trevor pred_dynamic = torch.argmax(outputs["dynamic"].detach(), 1).cpu() #.numpy() #same Trevor true_static = torch.squeeze( inputs["static"], 1).detach().cpu() #.numpy() #same Trevor true_dynamic = torch.squeeze( inputs["dynamic"], 1).detach().cpu() #.numpy() #same Trevor print("pred shape", pred_static.shape, "true shape", true_static.shape, pred_static.dtype, true_static.dtype) threat_static += compute_ts_road_map(pred_static.float(), true_static.float()) threat_dynamic += compute_ts_road_map(pred_dynamic.float(), true_dynamic.float()) for bb in range(pred_static.shape[0]): total_images += 1 iou_static += mean_IU(pred_static[bb].numpy(), true_static[bb].numpy()) iou_dynamic += mean_IU(pred_dynamic[bb].numpy(), true_dynamic[bb].numpy()) mAP_static += mean_precision(pred_static[bb].numpy(), true_static[bb].numpy()) mAP_dynamic += mean_precision(pred_dynamic[bb].numpy(), true_dynamic[bb].numpy()) print("total images", total_images) print(len(self.val_loader)) iou_static /= total_images #len(self.val_loader) mAP_static /= total_images #len(self.val_loader) threat_static /= total_images #len(self.val_loader) iou_dynamic /= total_images #len(self.val_loader) mAP_dynamic /= total_images #len(self.val_loader) threat_dynamic /= total_images #len(self.val_loader) # print("Epoch: %d | Validation: Static: mIOU: %.8f mAP: %.4f Dynamic: mIOU: %.8f mAP: %.4f"%(self.epoch, iou_static[1], mAP_static[1], iou_dynamic[1], mAP_dynamic[1])) print( "Epoch: %d | Validation: Static: mTS: %.8f mAP: %.4f Dynamic: mTS: %.8f mAP: %.4f iou_static %.8f iou_dynamic %.8f" % (self.epoch, threat_static, mAP_static[1], threat_dynamic, mAP_dynamic[1], iou_static[1], iou_dynamic[1])) return threat_static + threat_dynamic #may want to see about having two returns to save independently else: iou, mAP = np.array([0., 0.]), np.array([0., 0.]) threat = 0 for batch_idx, ipts in tqdm.tqdm(enumerate(self.val_loader)): with torch.no_grad(): inputs, outputs = self.process_batch(ipts, True) pred = torch.argmax(outputs["topview"].detach(), 1).cuda().numpy() true = torch.squeeze(inputs[self.opt.type], 1).detach().cuda().numpy() #print(pred.shape, true.shape) threat += compute_ts_road_map(pred, true) for bb in range(pred.shape[0]): # iou += mean_IU(pred[bb], true[bb]) mAP += mean_precision(pred[bb], true[bb]) iou /= len(self.val_loader) mAP /= len(self.val_loader) threat /= len(self.val_loader) # print("Epoch: %d | Validation: mIOU: %.4f mAP: %.4f"%(self.epoch, iou[1], mAP[1])) print("Epoch: %d | Validation: mTS: %.4f mAP: %.4f" % (self.epoch, threat, mAP[1])) return threat
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, i * len(sample), len(trainloader.dataset), 100. * i / len(trainloader), loss.item())) print("\n Average Train Epoch Loss for epoch {} is {} ", epoch+1, np.mean(train_losses)) #### validation logic #### model.eval() val_losses = [] threat_score = [] for i, (sample, target, road_img) in enumerate(valloader): sample = torch.stack(sample).to(device) batch_size = sample.shape[0] sample = sample.view(batch_size, -1, 256, 306) # size: ([3, 18, 256, 306]) road_img = torch.stack(road_img).float().to(device) with torch.no_grad(): pred_map = model(sample) loss = criterion(pred_map, road_img) val_losses.append(loss.item()) out_map = (pred_map > 0.5).float() threat_score.append(compute_ts_road_map(out_map, road_img).item()) print("Validation Epoch: {}, Average Validation Epoch Loss: {}".format(epoch, np.mean(val_losses))) print("Average Threat Score: {} ".format(np.mean(threat_score))) if np.mean(val_losses) < best_val_loss: best_val_loss = np.mean(val_losses) torch.save(model.state_dict(), 'test.pt')
def validation(epoch, batch_i, batch, param): with torch.no_grad(): outputs = None if not param['run_name'] in ['bbox_reg', 'bbox_reg_full']: inputs, labels = batch inputs = inputs.to(param['device']) labels = labels.to(param['device']) else: inputs, samples, labels, graph = batch inputs = inputs.to(param['device']) labels = labels.to(param['device']) samples = samples.to(param['device']) if param['run_name'] == 'mosaic': fusion_layer = param['model'][0].eval() static_model = param['model'][1].eval() fusion_outputs = fusion_layer(inputs) outputs = static_model(fusion_outputs).squeeze(1) elif param['run_name'] == 'polar': static_polar = param['model'].eval() outputs = static_polar(inputs.squeeze(1)).squeeze(1) elif param['run_name'] in ['front', 'bbox']: static_front = param['model'].eval() outputs = static_front(inputs).squeeze(1) for i in range(outputs.shape[0]): ts = compute_ts_road_map(outputs[i], labels[i]) param['ts'] += ts elif param['run_name'] in ['camerabased']: inputs = inputs.view(-1, 256, 16, 20) labels = labels.view(-1, 400, 538) static_camerabased = param['model'].eval() outputs = static_camerabased(inputs).squeeze(1) elif param['run_name'] == 'bbox_reg': camera_model = param['model'][0].eval() bbox_model = param['model'][1].eval() classifier = param['model'][2].eval() camera_feature = camera_model(inputs).unsqueeze(1) bbox_feature = bbox_model(samples.view(-1, 8)) # Batched bbox camera_feature_batch = camera_feature.repeat( 1, samples.shape[1], 1).view(-1, 1024) # repeat to match num of bbox features bbox_feature_positive = bbox_feature[labels.reshape(-1) == 1] camera_feature_batch_positive = camera_feature_batch[ labels.reshape(-1) == 1] labels = labels.reshape(-1)[labels.reshape(-1) == 1] outputs = classifier(camera_feature_batch_positive, bbox_feature_positive).squeeze(1) gen_bbox_heatmap(param, graph, camera_feature, epoch, batch_i) elif param['run_name'] == 'camerabased_full': inputs = inputs.view(-1, 256, 16, 20) labels = labels.view(-1, 800, 800) static_camerabased = param['model'][0] fusion_cameras = param['model'][1].eval() # [batch*6, 400, 538] outputs_cameras = static_camerabased(inputs).squeeze(1) # [batch, 6, 400, 538] outputs_cameras = outputs_cameras.view(-1, 6, 400, 538) # [batch, 800, 800] outputs = fusion_cameras(outputs_cameras).squeeze(1) elif param['run_name'] == 'camerabased_unet': inputs = inputs.view(-1, 256, 16, 20) labels = labels.view(-1, 800, 800) static_camerabased = param['model'][0] unet_cameras = param['model'][1].eval() # [batch*6, 400, 538] outputs_cameras = static_camerabased(inputs).squeeze(1) # [batch, 6, 400, 538] outputs_cameras = outputs_cameras.view(-1, 6, 400, 538) # [batch, 1, 800, 800] unet_input = concat_cameras( outputs_cameras.cpu().numpy()).unsqueeze(1).to(param['device']) # [batch, 800, 800] outputs = unet_cameras(unet_input).squeeze(1) loss = param['criterion'](outputs, labels.float()) param['running_loss'] += loss.item() if epoch % 5 == 0 and batch_i % 100 == 1 and param[ 'run_name'] != 'bbox_reg': sample_path = 'sample_output_{}'.format(param['run_name']) if not os.path.exists(sample_path): os.mkdir(sample_path) else: rand_camera = random.randrange(0, outputs.shape[0]) cv2.imwrite( '{}/val_sample_{}_{}_{}.png'.format( sample_path, epoch, batch_i, rand_camera), outputs[rand_camera].detach().cpu().numpy() * 255)
print("\n Average Train Epoch Loss for epoch {} is {} ", epoch + 1, np.mean(train_losses)) #### validation logic #### model.eval() val_losses = [] threat_score = [] for i, (sample, target, road_img) in enumerate(valloader): sample = torch.stack(sample).to(device) batch_size = sample.shape[0] sample = sample.view(batch_size, -1, 256, 306) # size: ([3, 18, 256, 306]) road_img = torch.stack(road_img).float().to(device) with torch.no_grad(): pred_map = model(sample) loss = criterion(pred_map, road_img) val_losses.append(loss.item()) out_map = (pred_map > 0.5).float() threat_score.append( compute_ts_road_map(out_map, road_img).item()) print("Validation Epoch: {}, Average Validation Epoch Loss: {}". format(epoch, np.mean(val_losses))) print("Average Threat Score: {} ".format(np.mean(threat_score))) if np.mean(val_losses) < best_val_loss: best_val_loss = np.mean(val_losses) torch.save(model.state_dict(), 'test.pt')