def evaluate_results(self, image, cage, mask, resulting_cage, gt_mask, results_file='results_cages'): utils.mkdir(results_file) result_file = results_file + "/" + cage.save_name if not resulting_cage: print 'No convergence reached for the cac-segmenter' else: resulting_cage.save_cage(result_file) res_fold = results_file + "/" + 'result' + cage.spec_name.split( "cage_")[-1] + '.png' result_mask = utils.create_ground_truth(cage, resulting_cage, mask) if result_mask: result_mask.save_image(filename=res_fold) if gt_mask: sorensen_dice_coeff = utils.sorensen_dice_coefficient( gt_mask, result_mask) TP, TN, FP, FN = utils.evaluate_segmentation( gt_mask, result_mask) print 'Sorensen-Dice coefficient', sorensen_dice_coeff print 'TP:', TP, 'TN:', TN, 'FP:', FP, 'FN:', FN return sorensen_dice_coeff, TP, TN, FP, FN
def evaluate_results(self, image, cage, mask, resulting_cage, gt_mask, results_file='results_cages'): utils.mkdir(results_file) result_file = results_file + "/" + cage.save_name if not resulting_cage: print 'No convergence reached for the cac-segmenter' else: resulting_cage.save_cage(result_file) res_fold = results_file + "/" + 'result' + cage.spec_name.split("cage_")[-1] + '.png' result_mask = utils.create_ground_truth(cage, resulting_cage, mask) if result_mask: result_mask.save_image(filename=res_fold) if gt_mask: sorensen_dice_coeff = utils.sorensen_dice_coefficient(gt_mask, result_mask) TP, TN, FP, FN = utils.evaluate_segmentation(gt_mask, result_mask) print 'Sorensen-Dice coefficient', sorensen_dice_coeff print 'TP:', TP, 'TN:', TN, 'FP:', FP, 'FN:', FN return sorensen_dice_coeff, TP, TN, FP, FN
gt = load_image( val_output_names[ind])[:args.crop_height, :args.crop_width] gt = helpers.reverse_one_hot( helpers.one_hot_it(gt, label_values)) # st = time.time() output_image = sess.run(network, feed_dict={net_input: input_image}) output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) out_vis_image = helpers.colour_code_segmentation( output_image, label_values) accuracy, class_accuracies, prec, rec, f1, iou = utils.evaluate_segmentation( pred=output_image, label=gt, num_classes=num_classes) file_name = utils.filepath_to_name(val_input_names[ind]) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec) f1_list.append(f1) iou_list.append(iou)
def main(): # transform = myTransform.Compose([myTransform.Resize(512, Image.BILINEAR), myTransform.ToTensor()]) co_transform = MyCoTransform(augment=True)#1024) co_transform_val = MyCoTransform(augment=False)#1024) root = "/home/jjin/adl/cityscapes_dataset" dataset_train = cityscapes(root, co_transform) loader_train = DataLoader(dataset_train, batch_size=4, shuffle=True,num_workers=4) dataset_val = cityscapes(root, co_transform_val, subset='val') loader_val = DataLoader(dataset_val, batch_size=1, shuffle=False,num_workers=4) NUM_CLASSES = 20 weight = torch.ones(NUM_CLASSES) weight[0] = 2.8149201869965 weight[1] = 6.9850029945374 weight[2] = 3.7890393733978 weight[3] = 9.9428062438965 weight[4] = 9.7702074050903 weight[5] = 9.5110931396484 weight[6] = 10.311357498169 weight[7] = 10.026463508606 weight[8] = 4.6323022842407 weight[9] = 9.5608062744141 weight[10] = 7.8698215484619 weight[11] = 9.5168733596802 weight[12] = 10.373730659485 weight[13] = 6.6616044044495 weight[14] = 10.260489463806 weight[15] = 10.287888526917 weight[16] = 10.289801597595 weight[17] = 10.405355453491 weight[18] = 10.138095855713 weight[19] = 0 weight = weight.cuda() criterion = CrossEntropyLoss2d(weight) print(type(criterion)) model = Net(NUM_CLASSES).cuda() # print(model) optimizer = Adam(model.parameters(), 0.001, (0.9, 0.999), eps=1e-08, weight_decay=1e-4) ## scheduler 2 savedir = '/home/jjin/adl/myImplementation/datasets/save' automated_log_path = savedir + "/automated_log_encoder.txt" modeltxtpath = savedir + "/model_encoder.txt" start_epoch = 1 iteration = 1 def load_my_state_dict(model, state_dict): #custom function to load model when not all dict elements own_state = model.state_dict() for name, param in state_dict.items(): if name not in own_state: continue own_state[name].copy_(param) return model weightspath = '/home/jjin/adl/myImplementation/datasets/trained_models/erfnet_pretrained.pth' for epoch in range(start_epoch, 10): # model.train() # for step, (images, labels) in enumerate(loader_train): # images = images.cuda() # labels = labels.cuda() # inputs = Variable(images) # targets = Variable(labels).long() # outputs = model(inputs) # optimizer.zero_grad() # loss = criterion(outputs, targets[:, 0]) # loss.backward() # optimizer.step() # iteration = iteration + 1 # pred = outputs.data.max(1)[1].cpu().numpy().flatten() # gt = labels.data.cpu().numpy().flatten() # global_accuracy, class_accuracies, prec, rec, f1, iou = evaluate_segmentation(pred, gt, NUM_CLASSES) # print('Epoch {} [{}/{}] Train_loss:{}'.format(epoch, step, len(loader_train), loss.data[0])) # loss.item() = loss.data[0] # torch.save(model.state_dict(), '{}_{}.pth'.format(os.path.join("/home/jjin/adl/myImplementation/datasets/save","model"),str(epoch))) # # model.load_state_dict(torch.load('/home/jjin/adl/myImplementation/datasets/trained_models/erfnet_pretrained.pth')) model = load_my_state_dict(model, torch.load(weightspath)) model.eval() with torch.no_grad(): for step_val, (images_val, labels_val) in enumerate(loader_val): images_val = images_val.cuda() labels_val = labels_val.cuda() inputs_val = Variable(images_val) targets_val = Variable(labels_val).long() outputs_val = model(inputs_val) loss_val = criterion(outputs_val, targets_val[:, 0]) # time_val.append(time.time() - start_time) pred = outputs_val.data.max(1)[1].cpu().numpy().flatten() gt = labels_val.data.cpu().numpy().flatten() global_accuracy, class_accuracies, prec, rec, f1, iou = evaluate_segmentation(pred, gt, NUM_CLASSES) print('Epoch {} [{}/{}] val_loss:{}'.format(epoch, step_val, len(loader_val), loss_val.data[0])) # loss.item() = loss.data[0]
def val_out(self, sess, val_init, threshold=0.5, output_dir="val", epoch=0): print("validation starts.") save_dir = output_dir + "/%d" % (epoch) if not os.path.exists(save_dir): os.mkdir(save_dir) target = open(save_dir + "/val_scores.csv", 'w') target.write( "val_name, avg_accuracy, precision, recall, f1 score, mean iou, %s\n" % (self.name_string)) sess.run(val_init) #for ind in range(self.num_val): scores_list = [] class_scores_list = [] precision_list = [] recall_list = [] f1_list = [] iou_list = [] try: while True: img, ann, output_image = sess.run( [self.img, self.mask, self.logits]) img = img[0, :, :, :] * 255 ann = np.array(ann[0, :, :, :]) ann = reverse_one_hot(ann) path, size = sess.run([self.path, self.size]) size = (size[0][0], size[0][1]) output_single_image = np.array(output_image) output_single_image = np.array(output_single_image[0, :, :, :]) output_image = reverse_one_hot(output_single_image) out_vis_image = colour_code_segmentation( output_image, self.label_values) accuracy, class_accuracies, prec, rec, f1, iou = evaluate_segmentation( pred=output_image, label=ann, num_classes=self.num_classes) dir = path[0].decode('ascii') file_name = filepath_to_name(dir) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") mask = colour_code_segmentation(ann, self.label_values) mask = cv2.cvtColor(np.uint8(mask), cv2.COLOR_RGB2BGR) out_vis_image = cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR) mask, ori_out_vis = resizeImage(mask, out_vis_image, size) out_vis_image = cv2.resize(ori_out_vis[:, :, 1], size, interpolation=cv2.INTER_NEAREST) out_vis_image[out_vis_image < threshold * 255] = 0 out_vis_image[out_vis_image >= threshold * 255] = 255 save_ori_img = cv2.cvtColor(np.uint8(img), cv2.COLOR_RGB2BGR) save_ori_img = cv2.resize(save_ori_img, size, interpolation=cv2.INTER_NEAREST) transparent_image = np.append(np.array(save_ori_img)[:, :, 0:3], out_vis_image[:, :, None], axis=-1) # transparent_image = Image.fromarray(transparent_image) cv2.imwrite(save_dir + "/%s_img.jpg" % (file_name), save_ori_img) cv2.imwrite(save_dir + "/%s_ann.png" % (file_name), mask) cv2.imwrite(save_dir + "/%s_ori_pred.png" % (file_name), ori_out_vis) cv2.imwrite(save_dir + "/%s_filter_pred.png" % (file_name), out_vis_image) cv2.imwrite(save_dir + "/%s_mat.png" % (file_name), transparent_image) scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec) f1_list.append(f1) iou_list.append(iou) except tf.errors.OutOfRangeError: avg_score = np.mean(scores_list) class_avg_scores = np.mean(class_scores_list, axis=0) avg_precision = np.mean(precision_list) avg_recall = np.mean(recall_list) avg_f1 = np.mean(f1_list) avg_iou = np.mean(iou_list) print("\nAverage validation accuracy for epoch # %04d = %f" % (epoch, avg_score)) print("Average per class validation accuracies for epoch # %04d:" % (epoch)) for index, item in enumerate(class_avg_scores): print("%s = %f" % (self.name_list[index], item)) print("Validation precision = ", avg_precision) print("Validation recall = ", avg_recall) print("Validation F1 score = ", avg_f1) print("Validation IoU score = ", avg_iou)
# Do the validation on a small set of validation images for ind in val_indices: input_image = np.expand_dims(np.float32(load_image(val_input_names[ind])[:args.crop_height, :args.crop_width]),axis=0)/255.0 gt = load_image(val_output_names[ind])[:args.crop_height, :args.crop_width] gt_tmp = helpers.one_hot_it(label=gt, label_values=label_values) gt_image = np.expand_dims(np.float32(gt_tmp),axis=0) gt = helpers.reverse_one_hot(gt_tmp) # st = time.time() #output_image = sess.run(network,feed_dict={net_input:input_image}) output_image ,current=sess.run([network,loss],feed_dict={net_input:input_image,net_output:gt_image}) output_image = np.array(output_image[0,:,:,:]) output_image = helpers.reverse_one_hot(output_image) out_vis_image = helpers.colour_code_segmentation(output_image, label_values) accuracy, class_accuracies, prec, rec, f1, iou = utils.evaluate_segmentation(pred=output_image, label=gt, num_classes=num_classes) file_name = utils.filepath_to_name(val_input_names[ind]) target.write("%s, %f, %.2f, %f, %f, %f, %f, %f"%(file_name, current,time.time()-val_st, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f"%(item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec) f1_list.append(f1) iou_list.append(iou) val_loss_list.append(current) #print ("CURRENT LOSS: ",current) ## add val loss
def val(): # Create directories if needed if not os.path.isdir(cfg.base_dir + "%s/%s" % ("result", "Val")): os.makedirs(cfg.base_dir + "%s/%s" % ("result", "Val")) target = open(cfg.base_dir + "%s/%s/val_scores.csv" % ("result", "Val"), 'w') target.write( "val_name, avg_accuracy, precision, recall, f1 score, mean iou, %s\n" % (class_names_string)) scores_list = [] class_scores_list = [] precision_list = [] recall_list = [] f1_list = [] iou_list = [] run_times_list = [] # Run testing on ALL test images for ind in range(len(val_input_names)): sys.stdout.write("\rRunning test image %d / %d" % (ind + 1, len(val_input_names))) sys.stdout.flush() input_image = np.expand_dims(np.float32( dataset.load_image(val_input_names[ind])[:cfg.height, :cfg.width]), axis=0) / 255.0 gt = dataset.load_image(val_output_names[ind])[:cfg.height, :cfg.width] gt = helpers.reverse_one_hot(helpers.one_hot_it(gt, label_values)) st = time.time() output_image = sess.run(network, feed_dict={net_input: input_image}) run_times_list.append(time.time() - st) output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) out_vis_image = helpers.colour_code_segmentation( output_image, label_values) accuracy, class_accuracies, prec, rec, f1, iou = utils.evaluate_segmentation( pred=output_image, label=gt, num_classes=num_classes) file_name = utils.filepath_to_name(val_input_names[ind]) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec) f1_list.append(f1) iou_list.append(iou) gt = helpers.colour_code_segmentation(gt, label_values) cv2.imwrite( cfg.base_dir + "%s/%s/%s_pred.png" % ("result", "Val", file_name), cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR)) cv2.imwrite( cfg.base_dir + "%s/%s/%s_gt.png" % ("result", "Val", file_name), cv2.cvtColor(np.uint8(gt), cv2.COLOR_RGB2BGR)) target.close() avg_score = np.mean(scores_list) class_avg_scores = np.mean(class_scores_list, axis=0) avg_precision = np.mean(precision_list) avg_recall = np.mean(recall_list) avg_f1 = np.mean(f1_list) avg_iou = np.mean(iou_list) avg_time = np.mean(run_times_list) print("Average test accuracy = ", avg_score) print("Average per class test accuracies = \n") for index, item in enumerate(class_avg_scores): print("%s = %f" % (class_names_list[index], item)) print("Average precision = ", avg_precision) print("Average recall = ", avg_recall) print("Average F1 score = ", avg_f1) print("Average mean IoU score = ", avg_iou) print("Average run time = ", avg_time)
def train(): if cfg.class_balancing: print("Computing class weights for trainlabel ...") class_weights = utils.compute_class_weights( labels_dir=train_output_names, label_values=label_values) weights = tf.reduce_sum(class_weights * net_output, axis=-1) unweighted_loss = None unweighted_loss = tf.nn.softmax_cross_entropy_with_logits_v2( logits=network, labels=net_output) losses = unweighted_loss * class_weights else: losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=network, labels=net_output) loss = tf.reduce_mean(losses) opt = tf.train.AdamOptimizer(cfg.lr).minimize( loss, var_list=[var for var in tf.trainable_variables()]) sess.run(tf.global_variables_initializer()) utils.count_params() # If a pre-trained ResNet is required, load the weights. # This must be done AFTER the variables are initialized with sess.run(tf.global_variables_initializer()) if init_fn is not None: init_fn(sess) avg_scores_per_epoch = [] avg_loss_per_epoch = [] # Which validation images do we want val_indices = [] num_vals = min(cfg.num_val_images, len(val_input_names)) # Set random seed to make sure models are validated on the same validation images. # So you can compare the results of different models more intuitively. random.seed(16) val_indices = random.sample(range(0, len(val_input_names)), num_vals) # Do the training here for epoch in range(0, cfg.num_epochs): current_losses = [] cnt = 0 # Equivalent to shuffling id_list = np.random.permutation(len(train_input_names)) num_iters = int(np.floor(len(id_list) / cfg.batch_size)) st = time.time() epoch_st = time.time() for i in range(num_iters): # st=time.time() input_image_batch = [] output_image_batch = [] # Collect a batch of images for j in range(cfg.batch_size): index = i * cfg.batch_size + j id = id_list[index] input_image = dataset.load_image(train_input_names[id]) output_image = dataset.load_image(train_output_names[id]) h, w, _ = input_image.shape new_h, new_w = dataset.getTrainSize(h, w) with tf.device('/cpu:0'): input_image, output_image = dataset.data_augmentation( input_image, output_image, new_h, new_w) # Prep the data. Make sure the labels are in one-hot format input_image = np.float32(input_image) / 255.0 output_image = np.float32( helpers.one_hot_it(label=output_image, label_values=label_values)) input_image_batch.append( np.expand_dims(input_image, axis=0)) output_image_batch.append( np.expand_dims(output_image, axis=0)) # ***** THIS CAUSES A MEMORY LEAK AS NEW TENSORS KEEP GETTING CREATED ***** # input_image = tf.image.crop_to_bounding_box(input_image, offset_height=0, offset_width=0, # target_height=args.crop_height, target_width=args.crop_width).eval(session=sess) # output_image = tf.image.crop_to_bounding_box(output_image, offset_height=0, offset_width=0, # target_height=args.crop_height, target_width=args.crop_width).eval(session=sess) # ***** THIS CAUSES A MEMORY LEAK AS NEW TENSORS KEEP GETTING CREATED ***** # memory() # print(cfg.batch_size) if cfg.batch_size == 1: input_image_batch = input_image_batch[0] output_image_batch = output_image_batch[0] else: input_image_batch = np.squeeze( np.stack(input_image_batch, axis=1)) output_image_batch = np.squeeze( np.stack(output_image_batch, axis=1)) # print(input_image_batch.shape) # Do the training _, current = sess.run([opt, loss], feed_dict={ net_input: input_image_batch, net_output: output_image_batch }) current_losses.append(current) cnt = cnt + cfg.batch_size if cnt % 20 == 0: string_print = "Epoch = %d Count = %d Current_Loss = %.4f Time = %.2f" % ( epoch, cnt, current, time.time() - st) utils.LOG(string_print) st = time.time() mean_loss = np.mean(current_losses) avg_loss_per_epoch.append(mean_loss) # Create directories if needed if not os.path.isdir(cfg.base_dir + "%s/%s/%04d" % ("checkpoints", cfg.model, epoch)): os.makedirs(cfg.base_dir + "%s/%s/%04d" % ("checkpoints", cfg.model, epoch)) # Save latest checkpoint to same file name print("Saving latest checkpoint") saver.save(sess, model_checkpoint_name) if val_indices != 0 and epoch % cfg.checkpoint_step == 0: print("Saving checkpoint for this epoch") saver.save( sess, cfg.base_dir + "%s/%s/%04d/model.ckpt" % ("checkpoints", cfg.model, epoch)) if epoch % cfg.validation_step == 0: print("Performing validation") target = open( cfg.base_dir + "%s/%s/%04d/val_scores.csv" % ("checkpoints", cfg.model, epoch), 'w') target.write( "val_name, avg_accuracy, precision, recall, f1 score, mean iou, %s\n" % (class_names_string)) scores_list = [] class_scores_list = [] precision_list = [] recall_list = [] f1_list = [] iou_list = [] # Do the validation on a small set of validation images for ind in val_indices: input_image = dataset.load_image(val_input_names[ind]) output_image = dataset.load_image(val_output_names[ind]) h, w, _ = input_image.shape new_h, new_w = dataset.getTrainSize(h, w) input_image, output_image = utils.random_crop( input_image, output_image, new_h, new_w) input_image = np.expand_dims(np.float32(input_image), axis=0) / 255.0 gt = helpers.reverse_one_hot( helpers.one_hot_it(output_image, label_values)) # st = time.time() output_image = sess.run(network, feed_dict={net_input: input_image}) output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) out_vis_image = helpers.colour_code_segmentation( output_image, label_values) accuracy, class_accuracies, prec, rec, f1, iou = utils.evaluate_segmentation( pred=output_image, label=gt, num_classes=num_classes) file_name = utils.filepath_to_name(val_input_names[ind]) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec) f1_list.append(f1) iou_list.append(iou) gt = helpers.colour_code_segmentation(gt, label_values) file_name = os.path.basename(val_input_names[ind]) file_name = os.path.splitext(file_name)[0] cv2.imwrite( cfg.base_dir + "%s/%s/%04d/%s_pred.png" % ("checkpoints", cfg.model, epoch, file_name), cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR)) cv2.imwrite( cfg.base_dir + "%s/%s/%04d/%s_gt.png" % ("checkpoints", cfg.model, epoch, file_name), cv2.cvtColor(np.uint8(gt), cv2.COLOR_RGB2BGR)) target.close() avg_score = np.mean(scores_list) class_avg_scores = np.mean(class_scores_list, axis=0) avg_scores_per_epoch.append(avg_score) avg_precision = np.mean(precision_list) avg_recall = np.mean(recall_list) avg_f1 = np.mean(f1_list) avg_iou = np.mean(iou_list) print("\nAverage validation accuracy for epoch # %04d = %f" % (epoch, avg_score)) print("Average per class validation accuracies for epoch # %04d:" % (epoch)) for index, item in enumerate(class_avg_scores): print("%s = %f" % (class_names_list[index], item)) print("Validation precision = ", avg_precision) print("Validation recall = ", avg_recall) print("Validation F1 score = ", avg_f1) print("Validation IoU score = ", avg_iou) epoch_time = time.time() - epoch_st remain_time = epoch_time * (cfg.num_epochs - 1 - epoch) m, s = divmod(remain_time, 60) h, m = divmod(m, 60) if s != 0: train_time = "Remaining training time = %d hours %d minutes %d seconds\n" % ( h, m, s) else: train_time = "Remaining training time : Training completed.\n" utils.LOG(train_time) scores_list = [] utils.drawLine(range(cfg.num_epochs), avg_scores_per_epoch, cfg.base_dir + 'checkpoints/' + cfg.model + '/accuracy_vs_epochs.png', title='Average validation accuracy vs epochs', xlabel='Epoch', ylabel='Avg. val. accuracy') utils.drawLine(range(cfg.num_epochs), avg_loss_per_epoch, cfg.base_dir + 'checkpoints/' + cfg.model + '/loss_vs_epochs.png', title='Average loss vs epochs', xlabel='Epoch', ylabel='Current loss')