def run(dataset_type, entry, save_outputs, small_run, input_file, output_dir): dataset = load_data(channels_first=False, dataset_type=dataset_type) cnn_data = np.load(input_file) if entry is None: metric_list = [ "delta1", "delta2", "delta3", "rel_abs_diff", "rmse", "mse", "log10", "weight" ] metrics = np.zeros( (len(dataset) if not small_run else small_run, len(metric_list))) entry_list = [] outputs = [] for i in range(len(dataset)): if small_run and i == small_run: break print("Running {}[{}]".format(dataset_type, i)) entry_list.append(i) init = cnn_data[i, ...] gt = dataset[i]["depth_cropped"] pred = init * (torch.median(gt).item() / np.median(init)) pred_metrics = get_depth_metrics( torch.from_numpy(pred).float(), gt, torch.ones_like(gt)) for j, metric_name in enumerate(metric_list[:-1]): metrics[i, j] = pred_metrics[metric_name] metrics[i, -1] = torch.numel(gt) # Option to save outputs: if save_outputs: outputs.append(pred) if save_outputs: np.save( os.path.join( output_dir, "dorn_median_{}_outputs.npy".format(dataset_type)), np.concatenate(outputs, axis=0)) # Save metrics using pandas metrics_df = pd.DataFrame(data=metrics, index=entry_list, columns=metric_list) metrics_df.to_pickle(path=os.path.join( output_dir, "dorn_median_{}_metrics.pkl".format(dataset_type))) # Compute weighted averages: average_metrics = np.average(metrics_df.ix[:, :-1], weights=metrics_df.weight, axis=0) average_df = pd.Series(data=average_metrics, index=metric_list[:-1]) average_df.to_csv(os.path.join( output_dir, "dorn_median_{}_avg_metrics.csv".format(dataset_type)), header=True) print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rms', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(average_metrics[0], average_metrics[1], average_metrics[2], average_metrics[3], average_metrics[4], average_metrics[6])) else: raise NotImplementedError
def run(dataset_type, spad_file, densedepth_depth_file, hyper_string, sid_bins, alpha, beta, offset, intensity_ablation, vectorized, entry, save_outputs, small_run, output_dir): print("output dir: {}".format(output_dir)) safe_makedir(output_dir) # Load all the data: print("Loading SPAD data from {}".format(spad_file)) spad_dict = np.load(spad_file, allow_pickle=True).item() spad_data = spad_dict["spad"] intensity_data = spad_dict["intensity"] spad_config = spad_dict["config"] print("Loading depth data from {}".format(densedepth_depth_file)) depth_data = np.load(densedepth_depth_file) dataset = load_data(channels_first=True, dataset_type=dataset_type) # Read SPAD config and determine proper course of action dc_count = spad_config["dc_count"] ambient = spad_config["dc_count"] / spad_config["spad_bins"] use_intensity = spad_config["use_intensity"] use_squared_falloff = spad_config["use_squared_falloff"] lambertian = spad_config["lambertian"] use_poisson = spad_config["use_poisson"] min_depth = spad_config["min_depth"] max_depth = spad_config["max_depth"] print("ambient: ", ambient) print("dc_count: ", dc_count) print("use_intensity: ", use_intensity) print("use_squared_falloff:", use_squared_falloff) print("lambertian:", lambertian) print("spad_data.shape", spad_data.shape) print("depth_data.shape", depth_data.shape) print("intensity_data.shape", intensity_data.shape) sid_obj_init = SID(sid_bins, alpha, beta, offset) if entry is None: metric_list = [ "delta1", "delta2", "delta3", "rel_abs_diff", "rmse", "mse", "log10", "weight" ] metrics = np.zeros( (len(dataset) if not small_run else small_run, len(metric_list))) entry_list = [] outputs = [] times = [] for i in range(depth_data.shape[0]): if small_run and i == small_run: break entry_list.append(i) print("Evaluating {}[{}]".format(dataset_type, i)) spad = spad_data[i, ...] bin_edges = np.linspace(min_depth, max_depth, len(spad) + 1) bin_values = (bin_edges[1:] + bin_edges[:-1]) / 2 # spad = preprocess_spad_ambient_estimate(spad, min_depth, max_depth, # correct_falloff=use_squared_falloff, # remove_dc= dc_count > 0., # global_min_depth=np.min(depth_data), # n_std=1. if use_poisson else 0.01) # Rescale SPAD_data weights = np.ones_like(depth_data[i, 0, ...]) # Ablation study: Turn off intensity, even if spad has been simulated with it. if use_intensity and not intensity_ablation: weights = intensity_data[i, 0, ...] if dc_count > 0.: spad = remove_dc_from_spad_edge( spad, ambient=ambient, # grad_th=2*ambient) grad_th=5 * np.sqrt(2 * ambient)) # print(2*ambient) # print(5*np.sqrt(2*ambient)) if use_squared_falloff: if lambertian: spad = spad * bin_values**4 else: spad = spad * bin_values**2 # Scale SID object to maximize bin utilization nonzeros = np.nonzero(spad)[0] if nonzeros.size > 0: min_depth_bin = np.min(nonzeros) max_depth_bin = np.max(nonzeros) + 1 if max_depth_bin > len(bin_edges) - 2: max_depth_bin = len(bin_edges) - 2 else: min_depth_bin = 0 max_depth_bin = len(bin_edges) - 2 min_depth_pred = np.clip(bin_edges[min_depth_bin], a_min=1e-2, a_max=None) max_depth_pred = np.clip(bin_edges[max_depth_bin + 1], a_min=1e-2, a_max=None) # print(min_depth_pred) # print(max_depth_pred) sid_obj_pred = SID(sid_bins=sid_obj_init.sid_bins, alpha=min_depth_pred, beta=max_depth_pred, offset=0.) spad_rescaled = rescale_bins(spad[min_depth_bin:max_depth_bin + 1], min_depth_pred, max_depth_pred, sid_obj_pred) start = process_time() pred, t = image_histogram_match_variable_bin( depth_data[i, 0, ...], spad_rescaled, weights, sid_obj_init, sid_obj_pred, vectorized) times.append(process_time() - start) # break # Calculate metrics gt = dataset[i]["depth_cropped"].unsqueeze(0) # print(gt.dtype) # print(pred.shape) # print(pred[20:30, 20:30]) pred_metrics = get_depth_metrics( torch.from_numpy(pred).unsqueeze(0).unsqueeze(0).float(), gt, torch.ones_like(gt)) for j, metric_name in enumerate(metric_list[:-1]): metrics[i, j] = pred_metrics[metric_name] metrics[i, -1] = np.size(pred) # Option to save outputs: if save_outputs: outputs.append(pred) print("\tAvg RMSE = {}".format( np.mean(metrics[:i + 1, metric_list.index("rmse")]))) if save_outputs: np.save( os.path.join(output_dir, "densedepth_{}_outputs.npy".format(hyper_string)), np.array(outputs)) print("Avg Time: {}".format(np.mean(times))) # Save metrics using pandas metrics_df = pd.DataFrame(data=metrics, index=entry_list, columns=metric_list) metrics_df.to_pickle(path=os.path.join( output_dir, "densedepth_{}_metrics.pkl".format(hyper_string))) # Compute weighted averages: average_metrics = np.average(metrics_df.ix[:, :-1], weights=metrics_df.weight, axis=0) average_df = pd.Series(data=average_metrics, index=metric_list[:-1]) average_df.to_csv(os.path.join( output_dir, "densedepth_{}_avg_metrics.csv".format(hyper_string)), header=True) print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rmse', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(average_metrics[0], average_metrics[1], average_metrics[2], average_metrics[3], average_metrics[4], average_metrics[6])) print("wrote results to {} ({})".format(output_dir, hyper_string)) else: input_unbatched = dataset.get_item_by_id(entry) # for key in ["rgb", "albedo", "rawdepth", "spad", "mask", "rawdepth_orig", "mask_orig", "albedo_orig"]: # input_[key] = input_[key].unsqueeze(0) from torch.utils.data._utils.collate import default_collate data = default_collate([input_unbatched]) # Checks entry = data["entry"][0] i = int(entry) entry = entry if isinstance(entry, str) else entry.item() print("Evaluating {}[{}]".format(dataset_type, i)) # Rescale SPAD spad = spad_data[i, ...] spad_rescaled = rescale_bins(spad, min_depth, max_depth, sid_obj) print("spad_rescaled", spad_rescaled) weights = np.ones_like(depth_data[i, 0, ...]) if use_intensity: weights = intensity_data[i, 0, ...] # spad_rescaled = preprocess_spad_sid_gmm(spad_rescaled, sid_obj, use_squared_falloff, dc_count > 0.) # spad_rescaled = preprocess_spad_sid(spad_rescaled, sid_obj, use_squared_falloff, dc_count > 0. # ) if dc_count > 0.: spad_rescaled = remove_dc_from_spad( spad_rescaled, sid_obj.sid_bin_edges, sid_obj.sid_bin_values[:-2]**2, lam=1e1 if use_poisson else 1e-1, eps_rel=1e-5) if use_squared_falloff: spad_rescaled = spad_rescaled * sid_obj.sid_bin_values[:-2]**2 # print(spad_rescaled) pred, _ = image_histogram_match(depth_data[i, 0, ...], spad_rescaled, weights, sid_obj) # break # Calculate metrics gt = data["depth_cropped"] print(gt.shape) print(pred.shape) print(gt[:, :, 40, 60]) print(depth_data[i, 0, 40, 60]) print("before rmse: ", np.sqrt(np.mean((gt.numpy() - depth_data[i, 0, ...])**2))) before_metrics = get_depth_metrics( torch.from_numpy( depth_data[i, 0, ...]).unsqueeze(0).unsqueeze(0).float(), gt, torch.ones_like(gt)) pred_metrics = get_depth_metrics( torch.from_numpy(pred).unsqueeze(0).unsqueeze(0).float(), gt, torch.ones_like(gt)) if save_outputs: np.save( os.path.join( output_dir, "densedepth_{}[{}]_{}_out.npy".format( dataset_type, entry, hyper_string)), pred) print("before:") print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rmse', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(before_metrics["delta1"], before_metrics["delta2"], before_metrics["delta3"], before_metrics["rel_abs_diff"], before_metrics["rmse"], before_metrics["log10"])) print("after:") print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rmse', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(pred_metrics["delta1"], pred_metrics["delta2"], pred_metrics["delta3"], pred_metrics["rel_abs_diff"], pred_metrics["rmse"], pred_metrics["log10"]))
def get_metrics(pred, gt, mask): return get_depth_metrics(pred, gt, mask)
def get_metrics(pred, truth, gt): return get_depth_metrics(pred, truth, gt)
def main(model_path, crop, dataset_type, entry, save_outputs, output_dir, seed, small_run, device): # Load the data dataset = load_data(channels_first=False, dataset_type=dataset_type) # Load the model model = get_midas(model_path, device) init_randomness(seed) if entry is None: dataloader = DataLoader( dataset, batch_size=1, shuffle=False, num_workers=0, # needs to be 0 to not crash autograd profiler. pin_memory=True) # if eval_config["save_outputs"]: with torch.no_grad(): metric_list = [ "delta1", "delta2", "delta3", "rel_abs_diff", "rmse", "mse", "log10", "weight" ] metrics = np.zeros((len(dataset) if not small_run else small_run, len(metric_list))) entry_list = [] outputs = [] for i, data in enumerate(dataloader): # TESTING if small_run and i == small_run: break entry = data["entry"][0] entry = entry if isinstance(entry, str) else entry.item() entry_list.append(entry) print("Evaluating {}".format(data["entry"][0])) # pred, pred_metrics = model.evaluate(data, device) pred = midas_gt_predict( model, data["rgb"].cpu().numpy().squeeze(), data["depth_cropped"].cpu().numpy().squeeze(), crop, device) pred = torch.from_numpy(pred).unsqueeze(0).unsqueeze(0).float() pred_metrics = get_depth_metrics(pred, data["depth_cropped"], torch.ones_like(pred)) print(pred_metrics) for j, metric_name in enumerate(metric_list[:-1]): metrics[i, j] = pred_metrics[metric_name] metrics[i, -1] = torch.numel(pred) # Option to save outputs: if save_outputs: outputs.append(pred.cpu().numpy()) if save_outputs: np.save( os.path.join(output_dir, "midas_{}_outputs.npy".format(dataset_type)), np.concatenate(outputs, axis=0)) # Save metrics using pandas metrics_df = pd.DataFrame(data=metrics, index=entry_list, columns=metric_list) metrics_df.to_pickle(path=os.path.join( output_dir, "midas_{}_metrics.pkl".format(dataset_type))) # Compute weighted averages: average_metrics = np.average(metrics_df.ix[:, :-1], weights=metrics_df.weight, axis=0) average_df = pd.Series(data=average_metrics, index=metric_list[:-1]) average_df.to_csv(os.path.join( output_dir, "midas_{}_avg_metrics.csv".format(dataset_type)), header=True) print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rms', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(average_metrics[0], average_metrics[1], average_metrics[2], average_metrics[3], average_metrics[4], average_metrics[6])) print("wrote results to {}".format(output_dir)) else: input_unbatched = dataset.get_item_by_id(entry) # for key in ["rgb", "albedo", "rawdepth", "spad", "mask", "rawdepth_orig", "mask_orig", "albedo_orig"]: # input_[key] = input_[key].unsqueeze(0) from torch.utils.data._utils.collate import default_collate data = default_collate([input_unbatched]) # Checks entry = data["entry"][0] entry = entry if isinstance(entry, str) else entry.item() print("Entry: {}".format(entry)) # print("remove_dc: ", model.remove_dc) # print("use_intensity: ", model.use_intensity) # print("use_squared_falloff: ", model.use_squared_falloff) pred, pred_metrics, pred_weight = model.evaluate( data["rgb"].to(device), data["depth_cropped"].to(device), torch.ones_like(data["depth_cropped"]).to(device)) if save_outputs: np.save( os.path.join(output_dir, "{}_{}_out.npy".format(dataset_type, entry))) print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rms', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(pred_metrics["delta1"], pred_metrics["delta2"], pred_metrics["delta3"], pred_metrics["rel_abs_diff"], pred_metrics["rms"], pred_metrics["log10"]))
def analyze(data_dir, calibration_file, scenes, offsets, output_dir, bin_width_ps, bin_width_m, min_depth_bin, max_depth_bin, min_depth, max_depth, sid_obj_init, ambient_max_depth_bin, device): midas_model = get_midas(model_path="MiDaS/model.pt", device=device) fc_kinect, fc_spad, pc_kinect, pc_spad, rdc_kinect, rdc_spad, tdc_kinect, tdc_spad, \ RotationOfSpad, TranslationOfSpad = extract_camera_params(calibration_file) # print(fc_kinect) # print(fc_spad) RotationOfKinect = RotationOfSpad.T TranslationOfKinect = -TranslationOfSpad.dot(RotationOfSpad.T) for scene, offset in zip(scenes, offsets): print("Running {}...".format(scene)) rootdir = os.path.join(data_dir, scene) scenedir = os.path.join(output_dir, scene) safe_makedir(os.path.join(scenedir)) # Load all the SPAD and kinect data spad = load_spad(os.path.join(rootdir, "spad", "data_accum.mat")) # print(spad.shape) spad_relevant = spad[..., min_depth_bin:max_depth_bin] spad_single_relevant = np.sum(spad_relevant, axis=(0,1)) ambient_estimate = np.mean(spad_single_relevant[:ambient_max_depth_bin]) # Get ground truth depth gt_idx = np.argmax(spad, axis=2) gt_r = signal.medfilt(np.fliplr(np.flipud((gt_idx * bin_width_m).T)), kernel_size=5) mask = (gt_r >= min_depth).astype('float').squeeze() gt_z = r_to_z(gt_r, fc_spad) gt_z = undistort_img(gt_z, fc_spad, pc_spad, rdc_spad, tdc_spad) mask = np.round(undistort_img(mask, fc_spad, pc_spad, rdc_spad, tdc_spad)) # Nearest neighbor upsampling to reduce holes in output scale_factor = 2 gt_z_up = cv2.resize(gt_z, dsize=(scale_factor*gt_z.shape[0], scale_factor*gt_z.shape[1]), interpolation=cv2.INTER_NEAREST) mask_up = cv2.resize(mask, dsize=(scale_factor*mask.shape[0], scale_factor*mask.shape[1]), interpolation=cv2.INTER_NEAREST) # Get RGB and intensity rgb, rgb_cropped, intensity, crop = load_and_crop_kinect(rootdir) # print(crop) # Undistort rgb # rgb = undistort_img(rgb, fc_kinect, pc_kinect, rdc_kinect, tdc_kinect) # # Crop # rgb_cropped = rgb[crop[0]:crop[1], crop[2]:crop[3], :] # Intensity # intensity = rgb_cropped[:, :, 0] / 225. # Project GT depth and mask to RGB image coordinates and crop it. gt_z_proj, mask_proj = project_depth(gt_z_up, mask_up, (rgb.shape[0], rgb.shape[1]), fc_spad*scale_factor, fc_kinect, pc_spad*scale_factor, pc_kinect, RotationOfKinect, TranslationOfKinect/1e3) gt_z_proj_crop = gt_z_proj[crop[0]+offset[0]:crop[1]+offset[0], crop[2]+offset[1]:crop[3]+offset[1]] gt_z_proj_crop = signal.medfilt(gt_z_proj_crop, kernel_size=5) # mask_proj_crop = mask_proj[crop[0]+offset[0]:crop[1]+offset[0], # crop[2]+offset[1]:crop[3]+offset[1]] mask_proj_crop = (gt_z_proj_crop >= min_depth).astype('float').squeeze() # Process SPAD spad_sid, sid_obj_pred = preprocess_spad(spad_single_relevant, ambient_estimate, min_depth, max_depth, sid_obj_init) # Initialize with CNN z_init = midas_predict(midas_model, rgb_cropped/255., depth_range=(min_depth, max_depth), device=device) r_init = z_to_r(z_init, fc_kinect) # Histogram Match weights = intensity # r_pred, t = image_histogram_match(r_init, spad_sid, weights, sid_obj) r_pred, t = image_histogram_match_variable_bin(r_init, spad_sid, weights, sid_obj_init, sid_obj_pred) z_pred = r_to_z(r_pred, fc_kinect) # Save histograms for later inspection intermediates = { "init_index": t[0], "init_hist": t[1], "pred_index": t[2], "pred_hist": t[3], "T_count": t[4] } np.save(os.path.join(scenedir, "intermediates.npy"), intermediates) # Mean Match med_bin = get_hist_med(spad_sid) hist_med = sid_obj_init.sid_bin_values[med_bin.astype('int')] r_med_scaled = np.clip(r_init * hist_med/np.median(r_init), a_min=min_depth, a_max=max_depth) z_med_scaled = r_to_z(r_med_scaled, fc_kinect) # Find min and max depth across r and z separately # min_r = min(np.min(a) for a in [gt_r, r_init, r_pred, r_med_scaled]) # max_r = max(np.max(a) for a in [gt_r, r_init, r_pred, r_med_scaled]) # min_z = min(np.min(a) for a in [gt_z, z_init, z_pred, z_med_scaled, gt_z_proj, gt_z_proj_crop]) # max_z = max(np.max(a) for a in [gt_z, z_init, z_pred, z_med_scaled, gt_z_proj, gt_z_proj_crop]) # mins_and_maxes = { # "min_r": min_r, # "max_r": max_r, # "min_z": min_z, # "max_z": max_z # } min_max = {} for k, img in zip(["gt_r", "r_init", "r_pred", "r_med_scaled"], [gt_r, r_init, r_pred, r_med_scaled]): min_max[k] = (np.min(img), np.max(img)) for k, img in zip(["gt_z", "z_init", "z_pred", "z_med_scaled", "gt_z_proj", "gt_z_proj_crop"], [gt_z, z_init, z_pred, z_med_scaled, gt_z_proj, gt_z_proj_crop]): min_max[k] = (np.min(img), np.max(img)) np.save(os.path.join(scenedir, "mins_and_maxes.npy"), min_max) # Save to figures print("Saving figures...") # spad_single_relevant w/ ambient estimate plt.figure() plt.bar(range(len(spad_single_relevant)), spad_single_relevant, log=True) plt.title("spad_single_relevant".format(scene)) plt.axhline(y=ambient_estimate, color='r', linewidth=0.5) plt.tight_layout() plt.savefig(os.path.join(scenedir, "spad_single_relevant.pdf")) # gt_r and gt_z and gt_z_proj and gt_z_proj_crop and masks depth_imwrite(gt_r, os.path.join(scenedir, "gt_r")) depth_imwrite(gt_z, os.path.join(scenedir, "gt_z")) depth_imwrite(gt_z_proj, os.path.join(scenedir, "gt_z_proj")) depth_imwrite(gt_z_proj_crop, os.path.join(scenedir, "gt_z_proj_crop")) depth_imwrite(mask, os.path.join(scenedir, "mask")) depth_imwrite(mask_proj, os.path.join(scenedir, "mask_proj")) depth_imwrite(mask_proj_crop, os.path.join(scenedir, "mask_proj_crop")) depth_imwrite(intensity, os.path.join(scenedir, "intensity")) np.save(os.path.join(scenedir, "crop.npy"), crop) # spad_sid after preprocessing plt.figure() plt.bar(range(len(spad_sid)), spad_sid, log=True) plt.title("spad_sid") plt.tight_layout() plt.savefig(os.path.join(scenedir, "spad_sid.pdf")) # rgb, rgb_cropped, intensity cv2.imwrite(os.path.join(scenedir, "rgb.png"), cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)) cv2.imwrite(os.path.join(scenedir, "rgb_cropped.png"), cv2.cvtColor(rgb_cropped, cv2.COLOR_RGB2BGR)) # r_init, z_init, diff_maps depth_imwrite(r_init, os.path.join(scenedir, "r_init")) depth_imwrite(z_init, os.path.join(scenedir, "z_init")) # r_pred, z_pred, diff_maps depth_imwrite(r_pred, os.path.join(scenedir, "r_pred")) depth_imwrite(z_pred, os.path.join(scenedir, "z_pred")) # r_med_scaled, z_med_scaled, diff_maps depth_imwrite(r_med_scaled, os.path.join(scenedir, "r_med_scaled")) depth_imwrite(z_med_scaled, os.path.join(scenedir, "z_med_scaled")) plt.close('all') # Compute metrics print("Computing error metrics...") # z_init # z_init_resized = cv2.resize(z_init, gt_z.shape) init_metrics = get_depth_metrics(torch.from_numpy(z_init).float(), torch.from_numpy(gt_z_proj_crop).float(), torch.from_numpy(mask_proj_crop).float()) np.save(os.path.join(scenedir, "init_metrics.npy"), init_metrics) # z_pred # z_pred_resized = cv2.resize(z_pred, gt_z.shape) pred_metrics = get_depth_metrics(torch.from_numpy(z_pred).float(), torch.from_numpy(gt_z_proj_crop).float(), torch.from_numpy(mask_proj_crop).float()) np.save(os.path.join(scenedir, "pred_metrics.npy"), pred_metrics) # z_med_scaled # z_med_scaled_resized = cv2.resize(z_med_scaled, gt_z.shape) med_scaled_metrics = get_depth_metrics(torch.from_numpy(z_med_scaled).float(), torch.from_numpy(gt_z_proj_crop).float(), torch.from_numpy(mask_proj_crop).float()) np.save(os.path.join(scenedir, "med_scaled_metrics.npy"), med_scaled_metrics)
def run(dataset_type, spad_file, densedepth_depth_file, hyper_string, sid_bins, alpha, beta, offset, lam, eps_rel, n_std, entry, save_outputs, small_run, subsampling, output_dir): # Load all the data: print("Loading SPAD data from {}".format(spad_file)) spad_dict = np.load(spad_file, allow_pickle=True).item() spad_data = spad_dict["spad"] intensity_data = spad_dict["intensity"] spad_config = spad_dict["config"] print("Loading depth data from {}".format(densedepth_depth_file)) depth_data = np.load(densedepth_depth_file) dataset = load_data(channels_first=True, dataset_type=dataset_type) # Read SPAD config and determine proper course of action dc_count = spad_config["dc_count"] use_intensity = spad_config["use_intensity"] use_squared_falloff = spad_config["use_squared_falloff"] use_poisson = spad_config["use_poisson"] min_depth = spad_config["min_depth"] max_depth = spad_config["max_depth"] print("dc_count: ", dc_count) print("use_intensity: ", use_intensity) print("use_squared_falloff:", use_squared_falloff) print("spad_data.shape", spad_data.shape) print("depth_data.shape", depth_data.shape) print("intensity_data.shape", intensity_data.shape) sid_obj = SID(sid_bins, alpha, beta, offset) if entry is None: metric_list = [ "delta1", "delta2", "delta3", "rel_abs_diff", "rmse", "mse", "log10", "weight" ] print(len(dataset) // subsampling) metrics = np.zeros( (len(dataset) // subsampling + 1 if not small_run else small_run, len(metric_list))) entry_list = [] outputs = [] for i in range(depth_data.shape[0]): idx = i * subsampling if idx >= depth_data.shape[0] or (small_run and i >= small_run): break entry_list.append(idx) print("Evaluating {}[{}]".format(dataset_type, idx)) spad = spad_data[idx, ...] # spad = preprocess_spad_ambient_estimate(spad, min_depth, max_depth, # correct_falloff=use_squared_falloff, # remove_dc= dc_count > 0., # global_min_depth=np.min(depth_data), # n_std=1. if use_poisson else 0.01) # Rescale SPAD_data spad_rescaled = rescale_bins(spad, min_depth, max_depth, sid_obj) weights = np.ones_like(depth_data[idx, 0, ...]) if use_intensity: weights = intensity_data[idx, 0, ...] # spad_rescaled = preprocess_spad_sid_gmm(spad_rescaled, sid_obj, use_squared_falloff, dc_count > 0.) if dc_count > 0.: spad_rescaled = remove_dc_from_spad( spad_rescaled, sid_obj.sid_bin_edges, sid_obj.sid_bin_values[:-2]**2, lam=1e-1 if spad_config["use_poisson"] else 1e-1, eps_rel=1e-5) # spad_rescaled = remove_dc_from_spad_poisson(spad_rescaled, # sid_obj.sid_bin_edges, # lam=lam) # spad = remove_dc_from_spad_ambient_estimate(spad, # min_depth, max_depth, # global_min_depth=np.min(depth_data), # n_std=n_std) # print(spad[:10]) # print(spad) if use_squared_falloff: spad_rescaled = spad_rescaled * sid_obj.sid_bin_values[:-2]**2 # bin_edges = np.linspace(min_depth, max_depth, len(spad) + 1) # bin_values = (bin_edges[1:] + bin_edges[:-1])/2 # spad = spad * bin_values ** 2 # spad_rescaled = rescale_bins(spad, min_depth, max_depth, sid_obj) pred, _ = image_histogram_match(depth_data[idx, 0, ...], spad_rescaled, weights, sid_obj) # break # Calculate metrics gt = dataset[idx]["depth_cropped"].unsqueeze(0) # print(gt.dtype) # print(pred.shape) # print(pred[20:30, 20:30]) pred_metrics = get_depth_metrics( torch.from_numpy(pred).unsqueeze(0).unsqueeze(0).float(), gt, torch.ones_like(gt)) for j, metric_name in enumerate(metric_list[:-1]): metrics[i, j] = pred_metrics[metric_name] metrics[i, -1] = np.size(pred) # Option to save outputs: if save_outputs: outputs.append(pred) print("\tAvg RMSE = {}".format( np.mean(metrics[:i + 1, metric_list.index("rmse")]))) if save_outputs: np.save( os.path.join(output_dir, "densedepth_{}_outputs.npy".format(hyper_string)), np.array(outputs)) # Save metrics using pandas metrics_df = pd.DataFrame(data=metrics, index=entry_list, columns=metric_list) metrics_df.to_pickle(path=os.path.join( output_dir, "densedepth_{}_metrics.pkl".format(hyper_string))) # Compute weighted averages: average_metrics = np.average(metrics_df.ix[:, :-1], weights=metrics_df.weight, axis=0) average_df = pd.Series(data=average_metrics, index=metric_list[:-1]) average_df.to_csv(os.path.join( output_dir, "densedepth_{}_avg_metrics.csv".format(hyper_string)), header=True) print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rmse', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(average_metrics[0], average_metrics[1], average_metrics[2], average_metrics[3], average_metrics[4], average_metrics[6])) print("wrote results to {} ({})".format(output_dir, hyper_string)) else: input_unbatched = dataset.get_item_by_id(entry) # for key in ["rgb", "albedo", "rawdepth", "spad", "mask", "rawdepth_orig", "mask_orig", "albedo_orig"]: # input_[key] = input_[key].unsqueeze(0) from torch.utils.data._utils.collate import default_collate data = default_collate([input_unbatched]) # Checks entry = data["entry"][0] i = int(entry) entry = entry if isinstance(entry, str) else entry.item() print("Evaluating {}[{}]".format(dataset_type, i)) # Rescale SPAD spad = spad_data[i, ...] spad_rescaled = rescale_bins(spad, min_depth, max_depth, sid_obj) print("spad_rescaled", spad_rescaled) weights = np.ones_like(depth_data[i, 0, ...]) if use_intensity: weights = intensity_data[i, 0, ...] # spad_rescaled = preprocess_spad_sid_gmm(spad_rescaled, sid_obj, use_squared_falloff, dc_count > 0.) # spad_rescaled = preprocess_spad_sid(spad_rescaled, sid_obj, use_squared_falloff, dc_count > 0. # ) if dc_count > 0.: spad_rescaled = remove_dc_from_spad( spad_rescaled, sid_obj.sid_bin_edges, sid_obj.sid_bin_values[:-2]**2, lam=1e1 if use_poisson else 1e-1, eps_rel=1e-5) if use_squared_falloff: spad_rescaled = spad_rescaled * sid_obj.sid_bin_values[:-2]**2 # print(spad_rescaled) pred, _ = image_histogram_match(depth_data[i, 0, ...], spad_rescaled, weights, sid_obj) # break # Calculate metrics gt = data["depth_cropped"] print(gt.shape) print(pred.shape) print(gt[:, :, 40, 60]) print(depth_data[i, 0, 40, 60]) print("before rmse: ", np.sqrt(np.mean((gt.numpy() - depth_data[i, 0, ...])**2))) before_metrics = get_depth_metrics( torch.from_numpy( depth_data[i, 0, ...]).unsqueeze(0).unsqueeze(0).float(), gt, torch.ones_like(gt)) pred_metrics = get_depth_metrics( torch.from_numpy(pred).unsqueeze(0).unsqueeze(0).float(), gt, torch.ones_like(gt)) if save_outputs: np.save( os.path.join( output_dir, "densedepth_{}[{}]_{}_out.npy".format( dataset_type, entry, hyper_string)), pred) print("before:") print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rmse', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(before_metrics["delta1"], before_metrics["delta2"], before_metrics["delta3"], before_metrics["rel_abs_diff"], before_metrics["rmse"], before_metrics["log10"])) print("after:") print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rmse', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(pred_metrics["delta1"], pred_metrics["delta2"], pred_metrics["delta3"], pred_metrics["rel_abs_diff"], pred_metrics["rmse"], pred_metrics["log10"]))
def analyze(figures_dir, data_dir, calibration_file, models, scenes, use_intensity, vectorized, device): kinect_intrinsics, spad_intrinsics, RotationOfSpad, TranslationOfSpad = extract_camera_params( calibration_file) RotationOfKinect = RotationOfSpad.T TranslationOfKinect = -TranslationOfSpad.dot(RotationOfSpad.T) for model_str, load_run in models.items(): model = load_run["load"](None, device) run_model = load_run["run"] output_dir = os.path.join(figures_dir, model_str) for scene, meta in scenes.items(): print("Running {}...".format(scene)) offset = meta["offset"] bin_width_ps = meta["bin_width_ps"] min_r = meta["min_r"] max_r = meta["max_r"] rootdir = os.path.join(data_dir, scene) scenedir = os.path.join(output_dir, scene) safe_makedir(os.path.join(scenedir)) bin_width_m = bin_width_ps * 3e8 / (2 * 1e12) min_depth_bin = np.floor(min_r / bin_width_m).astype('int') max_depth_bin = np.floor(max_r / bin_width_m).astype('int') # Compensate for z translation only min_depth = min_depth_bin * bin_width_m - TranslationOfSpad[2] / 1e3 # print(TranslationOfSpad) max_depth = (max_depth_bin + 1) * bin_width_m - TranslationOfSpad[2] / 1e3 sid_obj_init = SID(sid_bins=600, alpha=min_depth, beta=max_depth, offset=0) ambient_max_depth_bin = 100 # RGB from Kinect rgb, rgb_cropped, intensity, crop = load_and_crop_kinect( rootdir, kinect_file="kinect.mat") if not use_intensity: intensity = np.ones_like(intensity) # Load all the SPAD and kinect data spad = load_spad(os.path.join(rootdir, "spad", "data_accum.mat")) spad_relevant = spad[..., min_depth_bin:max_depth_bin] spad_single_relevant = np.sum(spad_relevant, axis=(0, 1)) ambient_estimate = np.mean( spad_single_relevant[:ambient_max_depth_bin]) np.save(os.path.join(scenedir, "spad_single_relevant.npy"), spad_single_relevant) # Get ground truth depth gt_idx = np.argmax(spad[..., :max_depth_bin], axis=2) gt_r = signal.medfilt(np.fliplr(np.flipud( (gt_idx * bin_width_m).T)), kernel_size=5) mask = (gt_r >= min_depth).astype('float').squeeze() gt_z = r_to_z(gt_r, spad_intrinsics["FocalLength"]) gt_z = undistort_img(gt_z, **spad_intrinsics) mask = np.round(undistort_img(mask, **spad_intrinsics)) # Nearest neighbor upsampling to reduce holes in output scale_factor = 2 gt_z_up = cv2.resize(gt_z, dsize=(scale_factor * gt_z.shape[0], scale_factor * gt_z.shape[1]), interpolation=cv2.INTER_NEAREST) mask_up = cv2.resize(mask, dsize=(scale_factor * mask.shape[0], scale_factor * mask.shape[1]), interpolation=cv2.INTER_NEAREST) # Project GT depth and mask to RGB image coordinates and crop it. gt_z_proj, mask_proj = project_depth( gt_z_up, mask_up, (rgb.shape[0], rgb.shape[1]), spad_intrinsics["FocalLength"] * scale_factor, kinect_intrinsics["FocalLength"], spad_intrinsics["PrincipalPoint"] * scale_factor, kinect_intrinsics["PrincipalPoint"], RotationOfKinect, TranslationOfKinect / 1e3) gt_z_proj_crop = gt_z_proj[crop[0] + offset[0]:crop[1] + offset[0], crop[2] + offset[1]:crop[3] + offset[1]] gt_z_proj_crop = signal.medfilt(gt_z_proj_crop, kernel_size=5) mask_proj_crop = (gt_z_proj_crop >= min_depth).astype('float').squeeze() # print("gt_z_proj_crop range:") # print(np.min(gt_z_proj_crop)) # print(np.max(gt_z_proj_crop)) # Process SPAD spad_sid, sid_obj_pred, spad_denoised, spad_corrected = \ preprocess_spad(spad_single_relevant, ambient_estimate, min_depth, max_depth, sid_obj_init) np.save(os.path.join(scenedir, "spad_denoised.npy"), spad_denoised) np.save(os.path.join(scenedir, "spad_corrected.npy"), spad_corrected) np.save(os.path.join(scenedir, "spad_sid.npy"), spad_sid) # Initialize with CNN z_init = run_model(model, rgb_cropped, depth_range=(min_depth, max_depth), device=device) print("min(z_init):", np.min(z_init)) print("max(z_init):", np.max(z_init)) r_init = z_to_r(z_init, kinect_intrinsics["FocalLength"]) print("min(r_init):", np.min(r_init)) print("max(r_init):", np.max(r_init)) # Histogram Match weights = intensity # weights = np.ones_like(r_init) # r_pred, t = image_histogram_match(r_init, spad_sid, weights, sid_obj) r_pred, t = image_histogram_match_variable_bin( r_init, spad_sid, weights, sid_obj_init, sid_obj_pred, vectorized) z_pred = r_to_z(r_pred, kinect_intrinsics["FocalLength"]) # print("z_pred range") # print(np.min(z_pred)) # print(np.max(z_pred)) # Save histograms for later inspection intermediates = { "init_index": t[0], "init_hist": t[1], "pred_index": t[2], "pred_hist": t[3], "T_count": t[4] } np.save(os.path.join(scenedir, "intermediates.npy"), intermediates) # Save processed SPAD data bin_edges = np.linspace(min_depth, max_depth, len(spad_single_relevant) + 1) bin_values = (bin_edges[1:] + bin_edges[:-1]) / 2 spad_metadata = { "ambient_estimate": ambient_estimate, "init_bin_edges": bin_edges, "init_bin_values": bin_values, "init_sid_bin_edges": sid_obj_init.sid_bin_edges, "init_sid_bin_values": sid_obj_init.sid_bin_values, "pred_sid_bin_edges": sid_obj_pred.sid_bin_edges, "pred_sid_bin_values": sid_obj_pred.sid_bin_values } np.save(os.path.join(scenedir, "spad_metadata.npy"), spad_metadata) # Mean Match med_bin = get_hist_med(spad_sid) hist_med = sid_obj_init.sid_bin_values[med_bin.astype('int')] r_med_scaled = np.clip(r_init * hist_med / np.median(r_init), a_min=min_depth, a_max=max_depth) z_med_scaled = r_to_z(r_med_scaled, kinect_intrinsics["FocalLength"]) min_max = {} for k, img in zip(["gt_r", "r_init", "r_pred", "r_med_scaled"], [gt_r, r_init, r_pred, r_med_scaled]): min_max[k] = (np.min(img), np.max(img)) for k, img in zip([ "gt_z", "z_init", "z_pred", "z_med_scaled", "gt_z_proj", "gt_z_proj_crop" ], [gt_z, z_init, z_pred, z_med_scaled, gt_z_proj, gt_z_proj_crop ]): min_max[k] = (np.min(img), np.max(img)) np.save(os.path.join(scenedir, "mins_and_maxes.npy"), min_max) # Save to figures print("Saving figures...") # spad_single_relevant w/ ambient estimate plt.figure() plt.bar(range(len(spad_single_relevant)), spad_single_relevant, log=True) plt.title("spad_single_relevant".format(scene)) plt.axhline(y=ambient_estimate, color='r', linewidth=0.5) plt.tight_layout() plt.savefig(os.path.join(scenedir, "spad_single_relevant.pdf")) # gt_r and gt_z and gt_z_proj and gt_z_proj_crop and masks depth_imwrite(gt_r, os.path.join(scenedir, "gt_r")) depth_imwrite(gt_z, os.path.join(scenedir, "gt_z")) depth_imwrite(gt_z_proj, os.path.join(scenedir, "gt_z_proj")) depth_imwrite(gt_z_proj_crop, os.path.join(scenedir, "gt_z_proj_crop")) depth_imwrite(mask, os.path.join(scenedir, "mask")) depth_imwrite(mask_proj, os.path.join(scenedir, "mask_proj")) depth_imwrite(mask_proj_crop, os.path.join(scenedir, "mask_proj_crop")) depth_imwrite(intensity, os.path.join(scenedir, "intensity")) np.save(os.path.join(scenedir, "crop.npy"), crop) # spad_sid after preprocessing plt.figure() plt.bar(range(len(spad_sid)), spad_sid, log=True) plt.title("spad_sid") plt.tight_layout() plt.savefig(os.path.join(scenedir, "spad_sid.pdf")) # rgb, rgb_cropped, intensity cv2.imwrite(os.path.join(scenedir, "rgb.png"), cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)) cv2.imwrite(os.path.join(scenedir, "rgb_cropped.png"), cv2.cvtColor(rgb_cropped, cv2.COLOR_RGB2BGR)) # r_init, z_init, diff_maps depth_imwrite(r_init, os.path.join(scenedir, "r_init")) depth_imwrite(z_init, os.path.join(scenedir, "z_init")) # r_pred, z_pred, diff_maps depth_imwrite(r_pred, os.path.join(scenedir, "r_pred")) depth_imwrite(z_pred, os.path.join(scenedir, "z_pred")) # r_med_scaled, z_med_scaled, diff_maps depth_imwrite(r_med_scaled, os.path.join(scenedir, "r_med_scaled")) depth_imwrite(z_med_scaled, os.path.join(scenedir, "z_med_scaled")) plt.close('all') # Compute metrics print("Computing error metrics...") # z_init # z_init_resized = cv2.resize(z_init, gt_z.shape) init_metrics = get_depth_metrics( torch.from_numpy(z_init).float(), torch.from_numpy(gt_z_proj_crop).float(), torch.from_numpy(mask_proj_crop).float()) np.save(os.path.join(scenedir, "init_metrics.npy"), init_metrics) # z_pred # z_pred_resized = cv2.resize(z_pred, gt_z.shape) pred_metrics = get_depth_metrics( torch.from_numpy(z_pred).float(), torch.from_numpy(gt_z_proj_crop).float(), torch.from_numpy(mask_proj_crop).float()) np.save(os.path.join(scenedir, "pred_metrics.npy"), pred_metrics) # z_med_scaled # z_med_scaled_resized = cv2.resize(z_med_scaled, gt_z.shape) med_scaled_metrics = get_depth_metrics( torch.from_numpy(z_med_scaled).float(), torch.from_numpy(gt_z_proj_crop).float(), torch.from_numpy(mask_proj_crop).float()) np.save(os.path.join(scenedir, "med_scaled_metrics.npy"), med_scaled_metrics)
def run(dataset_type, spad_file, dorn_depth_file, hyper_string, sid_bins, alpha, beta, offset, lam, eps_rel, entry, save_outputs, small_run, output_dir): # Load all the data: print("Loading SPAD data from {}".format(spad_file)) spad_dict = np.load(spad_file, allow_pickle=True).item() spad_data = spad_dict["spad"] intensity_data = spad_dict["intensity"] spad_config = spad_dict["config"] print("Loading depth data from {}".format(dorn_depth_file)) depth_data = np.load(dorn_depth_file) dataset = load_data(channels_first=True, dataset_type=dataset_type) # Read SPAD config and determine proper course of action dc_count = spad_config["dc_count"] ambient = spad_config["dc_count"] / spad_config["spad_bins"] use_intensity = spad_config["use_intensity"] use_squared_falloff = spad_config["use_squared_falloff"] min_depth = spad_config["min_depth"] max_depth = spad_config["max_depth"] print("dc_count: ", dc_count) print("use_intensity: ", use_intensity) print("use_squared_falloff:", use_squared_falloff) print("spad_data.shape", spad_data.shape) print("depth_data.shape", depth_data.shape) print("intensity_data.shape", intensity_data.shape) sid_obj = SID(sid_bins, alpha, beta, offset) if entry is None: metric_list = [ "delta1", "delta2", "delta3", "rel_abs_diff", "rmse", "mse", "log10", "weight" ] metrics = np.zeros( (len(dataset) if not small_run else small_run, len(metric_list))) entry_list = [] outputs = [] for i in range(depth_data.shape[0]): if small_run and i == small_run: break entry_list.append(i) print("Evaluating {}[{}]".format(dataset_type, i)) spad = spad_data[i, ...] weights = np.ones_like(depth_data[i, 0, ...]) if use_intensity: weights = intensity_data[i, 0, ...] if dc_count > 0.: spad = remove_dc_from_spad_edge(spad, ambient=ambient, grad_th=3 * ambient) if use_squared_falloff: bin_edges = np.linspace(min_depth, max_depth, len(spad) + 1) bin_values = (bin_edges[1:] + bin_edges[:-1]) / 2 spad = spad * bin_values**2 spad_rescaled = rescale_bins(spad, min_depth, max_depth, sid_obj) pred, _ = image_histogram_match(depth_data[i, 0, ...], spad_rescaled, weights, sid_obj) # break # Calculate metrics gt = dataset[i]["depth_cropped"].unsqueeze(0) # print(gt.dtype) # print(pred.shape) pred_metrics = get_depth_metrics( torch.from_numpy(pred).unsqueeze(0).unsqueeze(0).float(), gt, torch.ones_like(gt)) for j, metric_name in enumerate(metric_list[:-1]): metrics[i, j] = pred_metrics[metric_name] metrics[i, -1] = np.size(pred) # Option to save outputs: if save_outputs: outputs.append(pred) if save_outputs: np.save( os.path.join(output_dir, "dorn_{}_outputs.npy".format(hyper_string)), np.array(outputs)) # Save metrics using pandas metrics_df = pd.DataFrame(data=metrics, index=entry_list, columns=metric_list) metrics_df.to_pickle(path=os.path.join( output_dir, "dorn_{}_metrics.pkl".format(hyper_string))) # Compute weighted averages: average_metrics = np.average(metrics_df.ix[:, :-1], weights=metrics_df.weight, axis=0) average_df = pd.Series(data=average_metrics, index=metric_list[:-1]) average_df.to_csv(os.path.join( output_dir, "dorn_{}_avg_metrics.csv".format(hyper_string)), header=True) print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rms', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(average_metrics[0], average_metrics[1], average_metrics[2], average_metrics[3], average_metrics[4], average_metrics[6])) print("wrote results to {} ({})".format(output_dir, hyper_string)) else: input_unbatched = dataset.get_item_by_id(entry) # for key in ["rgb", "albedo", "rawdepth", "spad", "mask", "rawdepth_orig", "mask_orig", "albedo_orig"]: # input_[key] = input_[key].unsqueeze(0) from torch.utils.data._utils.collate import default_collate data = default_collate([input_unbatched]) # Checks entry = data["entry"][0] i = int(entry) entry = entry if isinstance(entry, str) else entry.item() print("Evaluating {}[{}]".format(dataset_type, i)) # Rescale SPAD spad_rescaled = rescale_bins(spad_data[i, ...], min_depth, max_depth, sid_obj) weights = np.ones_like(depth_data[i, 0, ...]) if use_intensity: weights = intensity_data[i, 0, ...] spad_rescaled = preprocess_spad(spad_rescaled, sid_obj, use_squared_falloff, dc_count > 0., lam=lam, eps_rel=eps_rel) pred, _ = image_histogram_match(depth_data[i, 0, ...], spad_rescaled, weights, sid_obj) # break # Calculate metrics gt = data["depth_cropped"] print(gt.shape) print(pred.shape) pred_metrics = get_depth_metrics( torch.from_numpy(pred).unsqueeze(0).unsqueeze(0), gt, torch.ones_like(gt)) if save_outputs: np.save( os.path.join( output_dir, "dorn_{}[{}]_{}_out.npy".format(dataset_type, entry, hyper_string))) print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format( 'd1', 'd2', 'd3', 'rel', 'rms', 'log_10')) print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}". format(pred_metrics["delta1"], pred_metrics["delta2"], pred_metrics["delta3"], pred_metrics["rel_abs_diff"], pred_metrics["rms"], pred_metrics["log10"]))