def eval_one(im_gt_pred_filepath, overwrite=False):
im_filepath, gt_filepath, pred_filepath = im_gt_pred_filepath
metrics_filepath = os.path.splitext(pred_filepath)[0] + ".metrics.json"
iou_filepath = os.path.splitext(pred_filepath)[0] + ".iou.json"
metrics = False
iou = False
if not overwrite:
# Try reading metrics and iou json
metrics = python_utils.load_json(metrics_filepath)
iou = python_utils.load_json(iou_filepath)
if not metrics or not iou:
# Have to compute at least one so load geometries
gt_polygons = load_geom(gt_filepath, im_filepath)
fixed_gt_polygons = polygon_utils.fix_polygons(
gt_polygons, buffer=0.0001
) # Buffer adds vertices but is needed to repair some geometries
pred_polygons = load_geom(pred_filepath, im_filepath)
fixed_pred_polygons = polygon_utils.fix_polygons(pred_polygons)
if not metrics:
# Compute and save metrics
max_angle_diffs = polygon_utils.compute_polygon_contour_measures(
fixed_pred_polygons,
fixed_gt_polygons,
sampling_spacing=1.0,
min_precision=0.5,
max_stretch=2,
progressbar=True)
max_angle_diffs = [
value for value in max_angle_diffs if value is not None
]
max_angle_diffs = np.array(max_angle_diffs)
max_angle_diffs = max_angle_diffs * 180 / np.pi # Convert to degrees
metrics = {"max_angle_diffs": list(max_angle_diffs)}
python_utils.save_json(metrics_filepath, metrics)
if not iou:
fixed_gt_polygon_collection = shapely.geometry.collection.GeometryCollection(
fixed_gt_polygons)
fixed_pred_polygon_collection = shapely.geometry.collection.GeometryCollection(
fixed_pred_polygons)
intersection = fixed_gt_polygon_collection.intersection(
fixed_pred_polygon_collection).area
union = fixed_gt_polygon_collection.union(
fixed_pred_polygon_collection).area
iou = {"intersection": intersection, "union": union}
python_utils.save_json(iou_filepath, iou)
return {
"metrics": metrics,
"iou": iou,
}
def eval_shapefile(gt_polygons, pred_info):
# Compute metrics
metrics_filepath = os.path.splitext(
pred_info["shapefile_filepath"])[0] + ".metrics.json"
metrics = python_utils.load_json(metrics_filepath)
if not metrics:
# Load pred shp
pred_polygons = load_shapefile(pred_info["shapefile_filepath"])
fixed_dt_polygons = polygon_utils.fix_polygons(pred_polygons)
print(f"Loaded {len(fixed_dt_polygons)} pred polygons")
max_angle_diffs = polygon_utils.compute_polygon_contour_measures(
fixed_dt_polygons,
gt_polygons,
sampling_spacing=1.0,
min_precision=0.5,
max_stretch=2)
max_angle_diffs = [
value for value in max_angle_diffs if value is not None
]
max_angle_diffs = np.array(max_angle_diffs)
max_angle_diffs = max_angle_diffs * 180 / np.pi # Convert to degrees
metrics = {"max_angle_diffs": list(max_angle_diffs)}
python_utils.save_json(metrics_filepath, metrics)
print(f"Got {len(metrics['max_angle_diffs'])} max_angle_diff values")
return metrics
def convert(in_filepath, stat_names):
stats = python_utils.load_json(in_filepath)
if stats:
string = ""
for stat_name in stat_names:
string += str(round(100 * stats[stat_name], 1)) + " & "
return string[:-2] + "\\\\"
else:
print("File not found!")
return ""
def load_image_ids(self):
image_id_list_filepath = os.path.join(self.processed_dir, "image_id_list-small.json" if self.small else "image_id_list.json")
if os.path.exists(image_id_list_filepath):
image_id_list = python_utils.load_json(image_id_list_filepath)
else:
coco = self.get_coco()
image_id_list = coco.getImgIds(catIds=coco.getCatIds())
# Save for later so that the whole coco object doesn't have to be instantiated when just reading processed samples with multiple workers:
python_utils.save_json(image_id_list_filepath, image_id_list)
return image_id_list
def get_stat_from_all(stat_filepath_format, method_info, tolerances,
stat_name):
stat_list = [0 for _ in tolerances]
for i, tolerance in enumerate(tolerances):
filepath = stat_filepath_format.format(method_info["name"], tolerance)
stats = python_utils.load_json(filepath)
if stats:
stat_list[i] = stats[stat_name]
else:
print_utils.print_warning(
"WARNING: could not open {}".format(filepath))
return stat_list
def load_raw_data(self, tile_info):
# Image:
tile_info["image"] = skimage.io.imread(tile_info["image_filepath"])
assert len(tile_info["image"].shape) == 3 and tile_info["image"].shape[
2] == 3, f"image should have shape (H, W, 3), not {tile_info['image'].shape}..."
# Annotations:
label_json = python_utils.load_json(tile_info["label_filepath"])
features_xy = label_json["features"]["xy"]
tile_info["gt_polygons"] = [
shapely.wkt.loads(obj["wkt"]) for obj in features_xy
]
return tile_info
def setup_run(config):
run_name = config["run_name"]
new_run = config["new_run"]
init_run_name = config["init_run_name"]
working_dir = os.path.dirname(os.path.abspath(__file__))
runs_dir = os.path.join(working_dir, config["runs_dirpath"])
# setup init checkpoints directory path if one is specified:
if init_run_name is not None:
init_run_dirpath = run_utils.setup_run_dir(runs_dir, init_run_name)
_, init_checkpoints_dirpath = run_utils.setup_run_subdirs(
init_run_dirpath)
else:
init_checkpoints_dirpath = None
# setup run directory:
run_dirpath = run_utils.setup_run_dir(runs_dir, run_name, new_run)
# save config in logs directory
run_utils.save_config(config, run_dirpath)
# save args
args_filepath = os.path.join(run_dirpath, "args.json")
args_to_save = {
"run_name": run_name,
"new_run": new_run,
"init_run_name": init_run_name,
"batch_size": config["optim_params"]["batch_size"],
}
if "samples" in config:
args_to_save["samples"] = config["samples"]
python_utils.save_json(args_filepath, args_to_save)
# save current commit hash
commit_hash = get_git_revision_hash()
if commit_hash is not None:
commit_hash_filepath = os.path.join(run_dirpath, "commit_history.json")
if os.path.exists(commit_hash_filepath):
commit_hashes = python_utils.load_json(commit_hash_filepath)
if commit_hashes[-1] != commit_hash:
commit_hashes.append(commit_hash)
python_utils.save_json(commit_hash_filepath, commit_hashes)
else:
commit_hashes = [commit_hash]
python_utils.save_json(commit_hash_filepath, commit_hashes)
return run_dirpath, init_checkpoints_dirpath
def load_raw_data(self, tile_info):
raw_data = {}
# Image:
raw_data["image_filepath"] = os.path.join(
self.root, self.raw_dirname, self.fold, IMAGE_DIRNAME,
IMAGE_FILENAME_FORMAT.format(city=tile_info["city"],
number=tile_info["number"]))
raw_data["image"] = skimage.io.imread(raw_data["image_filepath"])
assert len(raw_data["image"].shape) == 3 and raw_data["image"].shape[
2] == 3, f"image should have shape (H, W, 3), not {raw_data['image'].shape}..."
# Annotations:
if self.gt_source == "disk":
gt_base_filepath = os.path.join(
self.root, self.raw_dirname, self.fold, self.gt_dirname,
IMAGE_NAME_FORMAT.format(city=tile_info["city"],
number=tile_info["number"]))
gt_filepath = gt_base_filepath + "." + self.gt_type
if not os.path.exists(gt_filepath):
raw_data["gt_polygons"] = []
return raw_data
if self.gt_type == "npy":
np_gt_polygons = np.load(gt_filepath, allow_pickle=True)
gt_polygons = []
for np_gt_polygon in np_gt_polygons:
try:
gt_polygons.append(
shapely.geometry.Polygon(np_gt_polygon[:, ::-1]))
except ValueError:
# Invalid polygon, continue without it
continue
raw_data["gt_polygons"] = gt_polygons
elif self.gt_type == "geojson":
geojson = python_utils.load_json(gt_filepath)
raw_data["gt_polygons"] = list(shapely.geometry.shape(geojson))
elif self.gt_type == "tif":
raw_data["gt_polygons_image"] = skimage.io.imread(
gt_filepath)[:, :, None]
assert len(raw_data["gt_polygons_image"].shape) == 3 and raw_data["gt_polygons_image"].shape[2] == 1, \
f"Mask should have shape (H, W, 1), not {raw_data['gt_polygons_image'].shape}..."
elif self.gt_source == "osm":
raise NotImplementedError(
"Downloading from OSM is not implemented (takes too long to download, better download to disk first...)."
)
# np_gt_polygons = geo_utils.get_polygons_from_osm(image_filepath, tag="building", ij_coords=False)
return raw_data
def plot_metric(dirpath, info_list):
legend = []
for info in info_list:
metrics_filepath = os.path.join(dirpath, info["metrics_filepath"])
metrics = python_utils.load_json(metrics_filepath)
if metrics:
max_angle_diffs = np.array(metrics["max_angle_diffs"])
total = len(max_angle_diffs)
angle_thresholds = range(0, 91)
fraction_under_threshold_list = []
for angle_threshold in angle_thresholds:
fraction_under_threshold = np.sum(
max_angle_diffs < angle_threshold) / total
fraction_under_threshold_list.append(fraction_under_threshold)
# Plot
plt.plot(angle_thresholds, fraction_under_threshold_list)
# Compute mean
mean_error = np.mean(max_angle_diffs)
legend.append(f"{info['name']}: {mean_error:.1f}°")
else:
print_utils.print_warning("WARNING: could not open {}".format(
info["metrics_filepath"]))
plt.legend(legend, loc='lower right')
plt.xlabel("Threshold (degrees)")
plt.ylabel("Fraction of detections")
axes = plt.gca()
axes.set_xlim([0, 90])
axes.set_ylim([0, 1])
title = f"Cumulative max tangent angle error per detection"
plt.title(title)
plt.savefig(title.lower().replace(" ", "_") + ".pdf")
plt.show()
def launch_eval(args):
# --- Init: fills mode-specific default command-line arguments
if args.fold is None:
fold = {"test"} # Default value for eval mode
else:
fold = set(args.fold)
assert len(fold) == 1, "Argument 'fold' must be a single fold in eval mode"
# --- First step: figure out what run (experiment) is to be evaluated
# Option 1: the run_name argument is given in which case that's our run
run_name = None
config = None
if args.run_name is not None:
run_name = args.run_name
# Else option 2: Check if a config has been given to look for the run_name
if args.config is not None:
config = run_utils.load_config(args.config)
if config is not None and "run_name" in config and run_name is None:
run_name = config["run_name"]
# Else abort...
if run_name is None:
print_utils.print_error(
"ERROR: the run to evaluate could no be identified with the given arguments. "
"Please specify either the --run_name argument or the --config argument "
"linking to a config file that has a 'run_name' field filled with the name of "
"the run name to evaluate.")
sys.exit()
# --- Second step: get path to the run and if --config was not specified, load the config from the run's folder
run_dirpath = frame_field_learning.local_utils.get_run_dirpath(
args.runs_dirpath, run_name)
if config is None:
config = run_utils.load_config(config_dirpath=run_dirpath)
if config is None:
print_utils.print_error(
f"ERROR: the default run's config file at {run_dirpath} could not be loaded. "
f"Exiting now...")
sys.exit()
# --- Third step: Replace parameters in config file from command-line arguments
if args.dataset_params is not None:
config["dataset_params"] = python_utils.load_json(args.dataset_params)
if args.samples is not None:
config["samples"] = args.samples
if args.batch_size is not None:
config["optim_params"]["batch_size"] = args.batch_size
if args.eval_batch_size is not None:
config["optim_params"]["eval_batch_size"] = args.eval_batch_size
else:
config["optim_params"][
"eval_batch_size"] = 2 * config["optim_params"]["batch_size"]
config["fold"] = list(fold)
config["nodes"] = args.nodes
config["gpus"] = args.gpus
config["nr"] = args.nr
config["world_size"] = args.gpus * args.nodes
# --- Load params in config set as relative path to another JSON file
config = run_utils.load_defaults_in_config(
config, filepath_key="defaults_filepath")
config["eval_params"]["run_dirpath"] = run_dirpath
if args.eval_patch_size is not None:
config["eval_params"]["patch_size"] = args.eval_patch_size
if args.eval_patch_overlap is not None:
config["eval_params"]["patch_overlap"] = args.eval_patch_overlap
# Setup num_workers per process:
if config["num_workers"] is None:
config["num_workers"] = int(torch.multiprocessing.cpu_count() /
config["gpus"])
# --- Distributed init:
os.environ['MASTER_ADDR'] = args.master_addr
os.environ['MASTER_PORT'] = args.master_port
manager = torch.multiprocessing.Manager()
shared_dict = manager.dict()
shared_dict["name_list"] = manager.list()
shared_dict["iou_list"] = manager.list()
shared_dict["seg_coco_list"] = manager.list()
shared_dict["poly_coco_list"] = manager.list()
barrier = manager.Barrier(args.gpus)
torch.multiprocessing.spawn(eval_process,
nprocs=args.gpus,
args=(config, shared_dict, barrier))
def main():
from frame_field_learning import framefield, inference
import os
def save_gt_poly(raw_pred_filepath, name):
filapth_format = "/data/mapping_challenge_dataset/processed/val/data_{}.pt"
sample = torch.load(filapth_format.format(name))
polygon_arrays = sample["gt_polygons"]
polygons = [
shapely.geometry.Polygon(polygon[:, ::-1])
for polygon in polygon_arrays
]
base_filepath = os.path.join(os.path.dirname(raw_pred_filepath), name)
filepath = base_filepath + "." + name + ".pdf"
plot_utils.save_poly_viz(image, polygons, filepath)
config = {
"indicator_add_edge": False,
"steps": 500,
"data_level": 0.5,
"data_coef": 0.1,
"length_coef": 0.4,
"crossfield_coef": 0.5,
"poly_lr": 0.01,
"warmup_iters": 100,
"warmup_factor": 0.1,
"device": "cuda",
"tolerance": 0.5,
"seg_threshold": 0.5,
"min_area": 1,
"inner_polylines_params": {
"enable": False,
"max_traces": 1000,
"seed_threshold": 0.5,
"low_threshold": 0.1,
"min_width": 2, # Minimum width of trace to take into account
"max_width": 8,
"step_size": 1,
}
}
# --- Process args --- #
args = get_args()
if args.steps is not None:
config["steps"] = args.steps
if args.raw_pred is not None:
# Load raw_pred(s)
image_list = []
name_list = []
seg_list = []
crossfield_list = []
for raw_pred_filepath in args.raw_pred:
raw_pred = torch.load(raw_pred_filepath)
image_list.append(raw_pred["image"])
name_list.append(raw_pred["name"])
seg_list.append(raw_pred["seg"])
crossfield_list.append(raw_pred["crossfield"])
seg_batch = torch.stack(seg_list, dim=0)
crossfield_batch = torch.stack(crossfield_list, dim=0)
out_contours_batch, out_probs_batch = polygonize(
seg_batch, crossfield_batch, config)
for i, raw_pred_filepath in enumerate(args.raw_pred):
image = image_list[i]
name = name_list[i]
polygons = out_contours_batch[i]
base_filepath = os.path.join(os.path.dirname(raw_pred_filepath),
name)
filepath = base_filepath + ".poly_acm.pdf"
plot_utils.save_poly_viz(image, polygons, filepath)
# Load gt polygons
save_gt_poly(raw_pred_filepath, name)
elif args.im_filepath:
# Load from filepath, look for seg and crossfield next to the image
# Load data
image = skimage.io.imread(args.im_filepath)
base_filepath = os.path.splitext(args.im_filepath)[0]
seg = skimage.io.imread(base_filepath + ".seg.tif") / 255
crossfield = np.load(base_filepath + ".crossfield.npy",
allow_pickle=True)
# Select bbox for dev
if args.bbox is not None:
assert len(args.bbox) == 4, "bbox should have 4 values"
bbox = args.bbox
# bbox = [1440, 210, 1800, 650] # vienna12
# bbox = [2808, 2393, 3124, 2772] # innsbruck19
image = image[bbox[0]:bbox[2], bbox[1]:bbox[3]]
seg = seg[bbox[0]:bbox[2], bbox[1]:bbox[3]]
crossfield = crossfield[bbox[0]:bbox[2], bbox[1]:bbox[3]]
extra_name = ".bbox_{}_{}_{}_{}".format(*bbox)
else:
extra_name = ""
# Convert to torch and add batch dim
seg_batch = torch.tensor(np.transpose(seg[:, :, :2], (2, 0, 1)),
dtype=torch.float)[None, ...]
crossfield_batch = torch.tensor(np.transpose(crossfield, (2, 0, 1)),
dtype=torch.float)[None, ...]
out_contours_batch, out_probs_batch = polygonize(
seg_batch, crossfield_batch, config)
polygons = out_contours_batch[0]
# Save shapefile
# save_utils.save_shapefile(polygons, base_filepath + extra_name, "poly_acm", args.im_filepath)
# Save pdf viz
filepath = base_filepath + extra_name + ".poly_acm.pdf"
plot_utils.save_poly_viz(image,
polygons,
filepath,
linewidths=1,
draw_vertices=True,
color_choices=[[0, 1, 0, 1]])
elif args.dirpath:
seg_filename_list = fnmatch.filter(os.listdir(args.dirpath),
"*.seg.tif")
sorted(seg_filename_list)
pbar = tqdm(seg_filename_list, desc="Poly files")
for id, seg_filename in enumerate(pbar):
basename = seg_filename[:-len(".seg.tif")]
# shp_filepath = os.path.join(args.dirpath, basename + ".poly_acm.shp")
# Verify if image has already been polygonized
# if os.path.exists(shp_filepath):
# continue
pbar.set_postfix(name=basename, status="Loading data...")
crossfield_filename = basename + ".crossfield.npy"
metadata_filename = basename + ".metadata.json"
seg = skimage.io.imread(os.path.join(args.dirpath,
seg_filename)) / 255
crossfield = np.load(os.path.join(args.dirpath,
crossfield_filename),
allow_pickle=True)
metadata = python_utils.load_json(
os.path.join(args.dirpath, metadata_filename))
# image_filepath = metadata["image_filepath"]
# as_shp_filename = os.path.splitext(os.path.basename(image_filepath))[0]
# as_shp_filepath = os.path.join(os.path.dirname(os.path.dirname(image_filepath)), "gt_polygons", as_shp_filename + ".shp")
# Convert to torch and add batch dim
seg_batch = torch.tensor(np.transpose(seg[:, :, :2], (2, 0, 1)),
dtype=torch.float)[None, ...]
crossfield_batch = torch.tensor(np.transpose(
crossfield, (2, 0, 1)),
dtype=torch.float)[None, ...]
pbar.set_postfix(name=basename, status="Polygonazing...")
out_contours_batch, out_probs_batch = polygonize(
seg_batch, crossfield_batch, config)
polygons = out_contours_batch[0]
# Save as shp
# pbar.set_postfix(name=basename, status="Saving .shp...")
# geo_utils.save_shapefile_from_shapely_polygons(polygons, shp_filepath, as_shp_filepath)
# Save as COCO annotation
base_filepath = os.path.join(args.dirpath, basename)
inference.save_poly_coco(polygons, id, base_filepath,
"annotation.poly")
else:
print("Showcase on a very simple example:")
seg = np.zeros((6, 8, 3))
# Triangle:
seg[1, 4] = 1
seg[2, 3:5] = 1
seg[3, 2:5] = 1
seg[4, 1:5] = 1
# L extension:
seg[3:5, 5:7] = 1
u = np.zeros((6, 8), dtype=np.complex)
v = np.zeros((6, 8), dtype=np.complex)
# Init with grid
u.real = 1
v.imag = 1
# Add slope
u[:4, :4] *= np.exp(1j * np.pi / 4)
v[:4, :4] *= np.exp(1j * np.pi / 4)
# Add slope corners
# u[:2, 4:6] *= np.exp(1j * np.pi / 4)
# v[4:, :2] *= np.exp(- 1j * np.pi / 4)
crossfield = math_utils.compute_crossfield_c0c2(u, v)
seg_batch = torch.tensor(np.transpose(seg[:, :, :2], (2, 0, 1)),
dtype=torch.float)[None, ...]
crossfield_batch = torch.tensor(np.transpose(crossfield, (2, 0, 1)),
dtype=torch.float)[None, ...]
out_contours_batch, out_probs_batch = polygonize(
seg_batch, crossfield_batch, config)
polygons = out_contours_batch[0]
filepath = "demo_poly_acm.pdf"
plot_utils.save_poly_viz(seg,
polygons,
filepath,
linewidths=0.5,
draw_vertices=True,
crossfield=crossfield)
def __init__(self,
root: str,
fold: str = "train",
pre_process: bool = True,
patch_size: int = None,
pre_transform=None,
transform=None,
small: bool = False,
pool_size: int = 1,
raw_dirname: str = "raw",
processed_dirname: str = "processed"):
"""
@param root:
@param fold:
@param pre_process: If True, the dataset will be pre-processed first, saving training patches on disk. If False, data will be serve on-the-fly without any patching.
@param patch_size:
@param pre_transform:
@param transform:
@param small: If True, use a small subset of the dataset (for testing)
@param pool_size:
@param processed_dirname:
"""
self.root = root
self.fold = fold
self.pre_process = pre_process
self.patch_size = patch_size
self.pre_transform = pre_transform
self.transform = transform
self.small = small
if self.small:
print_utils.print_info(
"INFO: Using small version of the xView2 xBD dataset.")
self.pool_size = pool_size
self.raw_dirname = raw_dirname
if self.pre_process:
# Setup of pre-process
self.processed_dirpath = os.path.join(self.root, processed_dirname,
self.fold)
stats_filepath = os.path.join(
self.processed_dirpath,
"stats-small.pt" if self.small else "stats.pt")
processed_relative_paths_filepath = os.path.join(
self.processed_dirpath, "processed_paths-small.json"
if self.small else "processed_paths.json")
# Check if dataset has finished pre-processing by checking processed_relative_paths_filepath:
if os.path.exists(processed_relative_paths_filepath):
# Process done, load stats and processed_relative_paths
self.stats = torch.load(stats_filepath)
self.processed_relative_paths = python_utils.load_json(
processed_relative_paths_filepath)
else:
# Pre-process not finished, launch it:
tile_info_list = self.get_tile_info_list()
self.stats = self.process(tile_info_list)
# Save stats
torch.save(self.stats, stats_filepath)
# Save processed_relative_paths
self.processed_relative_paths = [
tile_info["processed_relative_filepath"]
for tile_info in tile_info_list
]
python_utils.save_json(processed_relative_paths_filepath,
self.processed_relative_paths)
else:
# Setup data sample list
self.tile_info_list = self.get_tile_info_list()
