def eval_one(im_gt_pred_filepath, overwrite=False): im_filepath, gt_filepath, pred_filepath = im_gt_pred_filepath metrics_filepath = os.path.splitext(pred_filepath)[0] + ".metrics.json" iou_filepath = os.path.splitext(pred_filepath)[0] + ".iou.json" metrics = False iou = False if not overwrite: # Try reading metrics and iou json metrics = python_utils.load_json(metrics_filepath) iou = python_utils.load_json(iou_filepath) if not metrics or not iou: # Have to compute at least one so load geometries gt_polygons = load_geom(gt_filepath, im_filepath) fixed_gt_polygons = polygon_utils.fix_polygons( gt_polygons, buffer=0.0001 ) # Buffer adds vertices but is needed to repair some geometries pred_polygons = load_geom(pred_filepath, im_filepath) fixed_pred_polygons = polygon_utils.fix_polygons(pred_polygons) if not metrics: # Compute and save metrics max_angle_diffs = polygon_utils.compute_polygon_contour_measures( fixed_pred_polygons, fixed_gt_polygons, sampling_spacing=1.0, min_precision=0.5, max_stretch=2, progressbar=True) max_angle_diffs = [ value for value in max_angle_diffs if value is not None ] max_angle_diffs = np.array(max_angle_diffs) max_angle_diffs = max_angle_diffs * 180 / np.pi # Convert to degrees metrics = {"max_angle_diffs": list(max_angle_diffs)} python_utils.save_json(metrics_filepath, metrics) if not iou: fixed_gt_polygon_collection = shapely.geometry.collection.GeometryCollection( fixed_gt_polygons) fixed_pred_polygon_collection = shapely.geometry.collection.GeometryCollection( fixed_pred_polygons) intersection = fixed_gt_polygon_collection.intersection( fixed_pred_polygon_collection).area union = fixed_gt_polygon_collection.union( fixed_pred_polygon_collection).area iou = {"intersection": intersection, "union": union} python_utils.save_json(iou_filepath, iou) return { "metrics": metrics, "iou": iou, }
def eval_shapefile(gt_polygons, pred_info): # Compute metrics metrics_filepath = os.path.splitext( pred_info["shapefile_filepath"])[0] + ".metrics.json" metrics = python_utils.load_json(metrics_filepath) if not metrics: # Load pred shp pred_polygons = load_shapefile(pred_info["shapefile_filepath"]) fixed_dt_polygons = polygon_utils.fix_polygons(pred_polygons) print(f"Loaded {len(fixed_dt_polygons)} pred polygons") max_angle_diffs = polygon_utils.compute_polygon_contour_measures( fixed_dt_polygons, gt_polygons, sampling_spacing=1.0, min_precision=0.5, max_stretch=2) max_angle_diffs = [ value for value in max_angle_diffs if value is not None ] max_angle_diffs = np.array(max_angle_diffs) max_angle_diffs = max_angle_diffs * 180 / np.pi # Convert to degrees metrics = {"max_angle_diffs": list(max_angle_diffs)} python_utils.save_json(metrics_filepath, metrics) print(f"Got {len(metrics['max_angle_diffs'])} max_angle_diff values") return metrics
def convert(in_filepath, stat_names): stats = python_utils.load_json(in_filepath) if stats: string = "" for stat_name in stat_names: string += str(round(100 * stats[stat_name], 1)) + " & " return string[:-2] + "\\\\" else: print("File not found!") return ""
def load_image_ids(self): image_id_list_filepath = os.path.join(self.processed_dir, "image_id_list-small.json" if self.small else "image_id_list.json") if os.path.exists(image_id_list_filepath): image_id_list = python_utils.load_json(image_id_list_filepath) else: coco = self.get_coco() image_id_list = coco.getImgIds(catIds=coco.getCatIds()) # Save for later so that the whole coco object doesn't have to be instantiated when just reading processed samples with multiple workers: python_utils.save_json(image_id_list_filepath, image_id_list) return image_id_list
def get_stat_from_all(stat_filepath_format, method_info, tolerances, stat_name): stat_list = [0 for _ in tolerances] for i, tolerance in enumerate(tolerances): filepath = stat_filepath_format.format(method_info["name"], tolerance) stats = python_utils.load_json(filepath) if stats: stat_list[i] = stats[stat_name] else: print_utils.print_warning( "WARNING: could not open {}".format(filepath)) return stat_list
def load_raw_data(self, tile_info): # Image: tile_info["image"] = skimage.io.imread(tile_info["image_filepath"]) assert len(tile_info["image"].shape) == 3 and tile_info["image"].shape[ 2] == 3, f"image should have shape (H, W, 3), not {tile_info['image'].shape}..." # Annotations: label_json = python_utils.load_json(tile_info["label_filepath"]) features_xy = label_json["features"]["xy"] tile_info["gt_polygons"] = [ shapely.wkt.loads(obj["wkt"]) for obj in features_xy ] return tile_info
def setup_run(config): run_name = config["run_name"] new_run = config["new_run"] init_run_name = config["init_run_name"] working_dir = os.path.dirname(os.path.abspath(__file__)) runs_dir = os.path.join(working_dir, config["runs_dirpath"]) # setup init checkpoints directory path if one is specified: if init_run_name is not None: init_run_dirpath = run_utils.setup_run_dir(runs_dir, init_run_name) _, init_checkpoints_dirpath = run_utils.setup_run_subdirs( init_run_dirpath) else: init_checkpoints_dirpath = None # setup run directory: run_dirpath = run_utils.setup_run_dir(runs_dir, run_name, new_run) # save config in logs directory run_utils.save_config(config, run_dirpath) # save args args_filepath = os.path.join(run_dirpath, "args.json") args_to_save = { "run_name": run_name, "new_run": new_run, "init_run_name": init_run_name, "batch_size": config["optim_params"]["batch_size"], } if "samples" in config: args_to_save["samples"] = config["samples"] python_utils.save_json(args_filepath, args_to_save) # save current commit hash commit_hash = get_git_revision_hash() if commit_hash is not None: commit_hash_filepath = os.path.join(run_dirpath, "commit_history.json") if os.path.exists(commit_hash_filepath): commit_hashes = python_utils.load_json(commit_hash_filepath) if commit_hashes[-1] != commit_hash: commit_hashes.append(commit_hash) python_utils.save_json(commit_hash_filepath, commit_hashes) else: commit_hashes = [commit_hash] python_utils.save_json(commit_hash_filepath, commit_hashes) return run_dirpath, init_checkpoints_dirpath
def load_raw_data(self, tile_info): raw_data = {} # Image: raw_data["image_filepath"] = os.path.join( self.root, self.raw_dirname, self.fold, IMAGE_DIRNAME, IMAGE_FILENAME_FORMAT.format(city=tile_info["city"], number=tile_info["number"])) raw_data["image"] = skimage.io.imread(raw_data["image_filepath"]) assert len(raw_data["image"].shape) == 3 and raw_data["image"].shape[ 2] == 3, f"image should have shape (H, W, 3), not {raw_data['image'].shape}..." # Annotations: if self.gt_source == "disk": gt_base_filepath = os.path.join( self.root, self.raw_dirname, self.fold, self.gt_dirname, IMAGE_NAME_FORMAT.format(city=tile_info["city"], number=tile_info["number"])) gt_filepath = gt_base_filepath + "." + self.gt_type if not os.path.exists(gt_filepath): raw_data["gt_polygons"] = [] return raw_data if self.gt_type == "npy": np_gt_polygons = np.load(gt_filepath, allow_pickle=True) gt_polygons = [] for np_gt_polygon in np_gt_polygons: try: gt_polygons.append( shapely.geometry.Polygon(np_gt_polygon[:, ::-1])) except ValueError: # Invalid polygon, continue without it continue raw_data["gt_polygons"] = gt_polygons elif self.gt_type == "geojson": geojson = python_utils.load_json(gt_filepath) raw_data["gt_polygons"] = list(shapely.geometry.shape(geojson)) elif self.gt_type == "tif": raw_data["gt_polygons_image"] = skimage.io.imread( gt_filepath)[:, :, None] assert len(raw_data["gt_polygons_image"].shape) == 3 and raw_data["gt_polygons_image"].shape[2] == 1, \ f"Mask should have shape (H, W, 1), not {raw_data['gt_polygons_image'].shape}..." elif self.gt_source == "osm": raise NotImplementedError( "Downloading from OSM is not implemented (takes too long to download, better download to disk first...)." ) # np_gt_polygons = geo_utils.get_polygons_from_osm(image_filepath, tag="building", ij_coords=False) return raw_data
def plot_metric(dirpath, info_list): legend = [] for info in info_list: metrics_filepath = os.path.join(dirpath, info["metrics_filepath"]) metrics = python_utils.load_json(metrics_filepath) if metrics: max_angle_diffs = np.array(metrics["max_angle_diffs"]) total = len(max_angle_diffs) angle_thresholds = range(0, 91) fraction_under_threshold_list = [] for angle_threshold in angle_thresholds: fraction_under_threshold = np.sum( max_angle_diffs < angle_threshold) / total fraction_under_threshold_list.append(fraction_under_threshold) # Plot plt.plot(angle_thresholds, fraction_under_threshold_list) # Compute mean mean_error = np.mean(max_angle_diffs) legend.append(f"{info['name']}: {mean_error:.1f}°") else: print_utils.print_warning("WARNING: could not open {}".format( info["metrics_filepath"])) plt.legend(legend, loc='lower right') plt.xlabel("Threshold (degrees)") plt.ylabel("Fraction of detections") axes = plt.gca() axes.set_xlim([0, 90]) axes.set_ylim([0, 1]) title = f"Cumulative max tangent angle error per detection" plt.title(title) plt.savefig(title.lower().replace(" ", "_") + ".pdf") plt.show()
def launch_eval(args): # --- Init: fills mode-specific default command-line arguments if args.fold is None: fold = {"test"} # Default value for eval mode else: fold = set(args.fold) assert len(fold) == 1, "Argument 'fold' must be a single fold in eval mode" # --- First step: figure out what run (experiment) is to be evaluated # Option 1: the run_name argument is given in which case that's our run run_name = None config = None if args.run_name is not None: run_name = args.run_name # Else option 2: Check if a config has been given to look for the run_name if args.config is not None: config = run_utils.load_config(args.config) if config is not None and "run_name" in config and run_name is None: run_name = config["run_name"] # Else abort... if run_name is None: print_utils.print_error( "ERROR: the run to evaluate could no be identified with the given arguments. " "Please specify either the --run_name argument or the --config argument " "linking to a config file that has a 'run_name' field filled with the name of " "the run name to evaluate.") sys.exit() # --- Second step: get path to the run and if --config was not specified, load the config from the run's folder run_dirpath = frame_field_learning.local_utils.get_run_dirpath( args.runs_dirpath, run_name) if config is None: config = run_utils.load_config(config_dirpath=run_dirpath) if config is None: print_utils.print_error( f"ERROR: the default run's config file at {run_dirpath} could not be loaded. " f"Exiting now...") sys.exit() # --- Third step: Replace parameters in config file from command-line arguments if args.dataset_params is not None: config["dataset_params"] = python_utils.load_json(args.dataset_params) if args.samples is not None: config["samples"] = args.samples if args.batch_size is not None: config["optim_params"]["batch_size"] = args.batch_size if args.eval_batch_size is not None: config["optim_params"]["eval_batch_size"] = args.eval_batch_size else: config["optim_params"][ "eval_batch_size"] = 2 * config["optim_params"]["batch_size"] config["fold"] = list(fold) config["nodes"] = args.nodes config["gpus"] = args.gpus config["nr"] = args.nr config["world_size"] = args.gpus * args.nodes # --- Load params in config set as relative path to another JSON file config = run_utils.load_defaults_in_config( config, filepath_key="defaults_filepath") config["eval_params"]["run_dirpath"] = run_dirpath if args.eval_patch_size is not None: config["eval_params"]["patch_size"] = args.eval_patch_size if args.eval_patch_overlap is not None: config["eval_params"]["patch_overlap"] = args.eval_patch_overlap # Setup num_workers per process: if config["num_workers"] is None: config["num_workers"] = int(torch.multiprocessing.cpu_count() / config["gpus"]) # --- Distributed init: os.environ['MASTER_ADDR'] = args.master_addr os.environ['MASTER_PORT'] = args.master_port manager = torch.multiprocessing.Manager() shared_dict = manager.dict() shared_dict["name_list"] = manager.list() shared_dict["iou_list"] = manager.list() shared_dict["seg_coco_list"] = manager.list() shared_dict["poly_coco_list"] = manager.list() barrier = manager.Barrier(args.gpus) torch.multiprocessing.spawn(eval_process, nprocs=args.gpus, args=(config, shared_dict, barrier))
def main(): from frame_field_learning import framefield, inference import os def save_gt_poly(raw_pred_filepath, name): filapth_format = "/data/mapping_challenge_dataset/processed/val/data_{}.pt" sample = torch.load(filapth_format.format(name)) polygon_arrays = sample["gt_polygons"] polygons = [ shapely.geometry.Polygon(polygon[:, ::-1]) for polygon in polygon_arrays ] base_filepath = os.path.join(os.path.dirname(raw_pred_filepath), name) filepath = base_filepath + "." + name + ".pdf" plot_utils.save_poly_viz(image, polygons, filepath) config = { "indicator_add_edge": False, "steps": 500, "data_level": 0.5, "data_coef": 0.1, "length_coef": 0.4, "crossfield_coef": 0.5, "poly_lr": 0.01, "warmup_iters": 100, "warmup_factor": 0.1, "device": "cuda", "tolerance": 0.5, "seg_threshold": 0.5, "min_area": 1, "inner_polylines_params": { "enable": False, "max_traces": 1000, "seed_threshold": 0.5, "low_threshold": 0.1, "min_width": 2, # Minimum width of trace to take into account "max_width": 8, "step_size": 1, } } # --- Process args --- # args = get_args() if args.steps is not None: config["steps"] = args.steps if args.raw_pred is not None: # Load raw_pred(s) image_list = [] name_list = [] seg_list = [] crossfield_list = [] for raw_pred_filepath in args.raw_pred: raw_pred = torch.load(raw_pred_filepath) image_list.append(raw_pred["image"]) name_list.append(raw_pred["name"]) seg_list.append(raw_pred["seg"]) crossfield_list.append(raw_pred["crossfield"]) seg_batch = torch.stack(seg_list, dim=0) crossfield_batch = torch.stack(crossfield_list, dim=0) out_contours_batch, out_probs_batch = polygonize( seg_batch, crossfield_batch, config) for i, raw_pred_filepath in enumerate(args.raw_pred): image = image_list[i] name = name_list[i] polygons = out_contours_batch[i] base_filepath = os.path.join(os.path.dirname(raw_pred_filepath), name) filepath = base_filepath + ".poly_acm.pdf" plot_utils.save_poly_viz(image, polygons, filepath) # Load gt polygons save_gt_poly(raw_pred_filepath, name) elif args.im_filepath: # Load from filepath, look for seg and crossfield next to the image # Load data image = skimage.io.imread(args.im_filepath) base_filepath = os.path.splitext(args.im_filepath)[0] seg = skimage.io.imread(base_filepath + ".seg.tif") / 255 crossfield = np.load(base_filepath + ".crossfield.npy", allow_pickle=True) # Select bbox for dev if args.bbox is not None: assert len(args.bbox) == 4, "bbox should have 4 values" bbox = args.bbox # bbox = [1440, 210, 1800, 650] # vienna12 # bbox = [2808, 2393, 3124, 2772] # innsbruck19 image = image[bbox[0]:bbox[2], bbox[1]:bbox[3]] seg = seg[bbox[0]:bbox[2], bbox[1]:bbox[3]] crossfield = crossfield[bbox[0]:bbox[2], bbox[1]:bbox[3]] extra_name = ".bbox_{}_{}_{}_{}".format(*bbox) else: extra_name = "" # Convert to torch and add batch dim seg_batch = torch.tensor(np.transpose(seg[:, :, :2], (2, 0, 1)), dtype=torch.float)[None, ...] crossfield_batch = torch.tensor(np.transpose(crossfield, (2, 0, 1)), dtype=torch.float)[None, ...] out_contours_batch, out_probs_batch = polygonize( seg_batch, crossfield_batch, config) polygons = out_contours_batch[0] # Save shapefile # save_utils.save_shapefile(polygons, base_filepath + extra_name, "poly_acm", args.im_filepath) # Save pdf viz filepath = base_filepath + extra_name + ".poly_acm.pdf" plot_utils.save_poly_viz(image, polygons, filepath, linewidths=1, draw_vertices=True, color_choices=[[0, 1, 0, 1]]) elif args.dirpath: seg_filename_list = fnmatch.filter(os.listdir(args.dirpath), "*.seg.tif") sorted(seg_filename_list) pbar = tqdm(seg_filename_list, desc="Poly files") for id, seg_filename in enumerate(pbar): basename = seg_filename[:-len(".seg.tif")] # shp_filepath = os.path.join(args.dirpath, basename + ".poly_acm.shp") # Verify if image has already been polygonized # if os.path.exists(shp_filepath): # continue pbar.set_postfix(name=basename, status="Loading data...") crossfield_filename = basename + ".crossfield.npy" metadata_filename = basename + ".metadata.json" seg = skimage.io.imread(os.path.join(args.dirpath, seg_filename)) / 255 crossfield = np.load(os.path.join(args.dirpath, crossfield_filename), allow_pickle=True) metadata = python_utils.load_json( os.path.join(args.dirpath, metadata_filename)) # image_filepath = metadata["image_filepath"] # as_shp_filename = os.path.splitext(os.path.basename(image_filepath))[0] # as_shp_filepath = os.path.join(os.path.dirname(os.path.dirname(image_filepath)), "gt_polygons", as_shp_filename + ".shp") # Convert to torch and add batch dim seg_batch = torch.tensor(np.transpose(seg[:, :, :2], (2, 0, 1)), dtype=torch.float)[None, ...] crossfield_batch = torch.tensor(np.transpose( crossfield, (2, 0, 1)), dtype=torch.float)[None, ...] pbar.set_postfix(name=basename, status="Polygonazing...") out_contours_batch, out_probs_batch = polygonize( seg_batch, crossfield_batch, config) polygons = out_contours_batch[0] # Save as shp # pbar.set_postfix(name=basename, status="Saving .shp...") # geo_utils.save_shapefile_from_shapely_polygons(polygons, shp_filepath, as_shp_filepath) # Save as COCO annotation base_filepath = os.path.join(args.dirpath, basename) inference.save_poly_coco(polygons, id, base_filepath, "annotation.poly") else: print("Showcase on a very simple example:") seg = np.zeros((6, 8, 3)) # Triangle: seg[1, 4] = 1 seg[2, 3:5] = 1 seg[3, 2:5] = 1 seg[4, 1:5] = 1 # L extension: seg[3:5, 5:7] = 1 u = np.zeros((6, 8), dtype=np.complex) v = np.zeros((6, 8), dtype=np.complex) # Init with grid u.real = 1 v.imag = 1 # Add slope u[:4, :4] *= np.exp(1j * np.pi / 4) v[:4, :4] *= np.exp(1j * np.pi / 4) # Add slope corners # u[:2, 4:6] *= np.exp(1j * np.pi / 4) # v[4:, :2] *= np.exp(- 1j * np.pi / 4) crossfield = math_utils.compute_crossfield_c0c2(u, v) seg_batch = torch.tensor(np.transpose(seg[:, :, :2], (2, 0, 1)), dtype=torch.float)[None, ...] crossfield_batch = torch.tensor(np.transpose(crossfield, (2, 0, 1)), dtype=torch.float)[None, ...] out_contours_batch, out_probs_batch = polygonize( seg_batch, crossfield_batch, config) polygons = out_contours_batch[0] filepath = "demo_poly_acm.pdf" plot_utils.save_poly_viz(seg, polygons, filepath, linewidths=0.5, draw_vertices=True, crossfield=crossfield)
def __init__(self, root: str, fold: str = "train", pre_process: bool = True, patch_size: int = None, pre_transform=None, transform=None, small: bool = False, pool_size: int = 1, raw_dirname: str = "raw", processed_dirname: str = "processed"): """ @param root: @param fold: @param pre_process: If True, the dataset will be pre-processed first, saving training patches on disk. If False, data will be serve on-the-fly without any patching. @param patch_size: @param pre_transform: @param transform: @param small: If True, use a small subset of the dataset (for testing) @param pool_size: @param processed_dirname: """ self.root = root self.fold = fold self.pre_process = pre_process self.patch_size = patch_size self.pre_transform = pre_transform self.transform = transform self.small = small if self.small: print_utils.print_info( "INFO: Using small version of the xView2 xBD dataset.") self.pool_size = pool_size self.raw_dirname = raw_dirname if self.pre_process: # Setup of pre-process self.processed_dirpath = os.path.join(self.root, processed_dirname, self.fold) stats_filepath = os.path.join( self.processed_dirpath, "stats-small.pt" if self.small else "stats.pt") processed_relative_paths_filepath = os.path.join( self.processed_dirpath, "processed_paths-small.json" if self.small else "processed_paths.json") # Check if dataset has finished pre-processing by checking processed_relative_paths_filepath: if os.path.exists(processed_relative_paths_filepath): # Process done, load stats and processed_relative_paths self.stats = torch.load(stats_filepath) self.processed_relative_paths = python_utils.load_json( processed_relative_paths_filepath) else: # Pre-process not finished, launch it: tile_info_list = self.get_tile_info_list() self.stats = self.process(tile_info_list) # Save stats torch.save(self.stats, stats_filepath) # Save processed_relative_paths self.processed_relative_paths = [ tile_info["processed_relative_filepath"] for tile_info in tile_info_list ] python_utils.save_json(processed_relative_paths_filepath, self.processed_relative_paths) else: # Setup data sample list self.tile_info_list = self.get_tile_info_list()