def predict_file(model, csv_file, root_dir, savedir, device, iou_threshold=0.1): """Create a dataset and predict entire annotation file Csv file format is .csv file with the columns "image_path", "xmin","ymin","xmax","ymax" for the image name and bounding box position. Image_path is the relative filename, not absolute path, which is in the root_dir directory. One bounding box per line. Args: csv_file: path to csv file root_dir: directory of images. If none, uses "image_dir" in config savedir: Optional. Directory to save image plots. device: pytorch device of 'cuda' or 'cpu' for gpu prediction. Set internally. Returns: df: pandas dataframe with bounding boxes, label and scores for each image in the csv file """ input_csv = pd.read_csv(csv_file) # Just predict each image once. images = input_csv["image_path"].unique() prediction_list = [] for path in images: image = io.imread("{}/{}".format(root_dir, path)) image = preprocess.preprocess_image(image) # Just predict the images, even though we have the annotations if not device.type == "cpu": image = image.to(device) prediction = model(image) prediction = visualize.format_boxes(prediction[0]) prediction = across_class_nms(prediction, iou_threshold=iou_threshold) prediction["image_path"] = path prediction_list.append(prediction) if savedir: #if on GPU, bring back to cpu for plotting # Just predict the images, even though we have the annotations if not device.type == "cpu": image = image.to("cpu") image = image.squeeze(0).permute(1, 2, 0) plot, ax = visualize.plot_predictions(image, prediction) annotations = input_csv[input_csv.image_path == path] plot = visualize.add_annotations(plot, ax, annotations) plot.savefig("{}/{}.png".format(savedir, os.path.splitext(path)[0]), dpi=300) df = pd.concat(prediction_list, ignore_index=True) return df
def predict_image(model, image, return_plot, device, iou_threshold=0.1): """Predict an image with a deepforest model Args: image: a numpy array of a RGB image ranged from 0-255 path: optional path to read image from disk instead of passing image arg return_plot: Return image with plotted detections device: pytorch device of 'cuda' or 'cpu' for gpu prediction. Set internally. Returns: boxes: A pandas dataframe of predictions (Default) img: The input with predictions overlaid (Optional) """ image = preprocess.preprocess_image(image) image = image.to(device) prediction = model(image) # return None for no predictions if len(prediction[0]["boxes"]) == 0: return None # This function on takes in a single image. df = visualize.format_boxes(prediction[0]) df = across_class_nms(df, iou_threshold=iou_threshold) if return_plot: # Matplotlib likes no batch dim and channels first image = image.squeeze(0).permute(1, 2, 0) plot, ax = visualize.plot_predictions(image, df) return plot else: return df
def evaluate_image(predictions, ground_df, root_dir, savedir=None): """ Compute intersection-over-union matching among prediction and ground truth boxes for one image Args: df: a pandas dataframe with columns name, xmin, xmax, ymin, ymax, label. The 'name' column should be the path relative to the location of the file. summarize: Whether to group statistics by plot and overall score image_coordinates: Whether the current boxes are in coordinate system of the image, e.g. origin (0,0) upper left. root_dir: Where to search for image names in df savedir: optional directory to save image with overlaid predictions and annotations Returns: result: pandas dataframe with crown ids of prediciton and ground truth and the IoU score. """ plot_names = predictions["image_path"].unique() if len(plot_names) > 1: raise ValueError( "More than one plot passed to image crown: {}".format(plot_name)) else: plot_name = plot_names[0] predictions['geometry'] = predictions.apply( lambda x: shapely.geometry.box(x.xmin, x.ymin, x.xmax, x.ymax), axis=1) predictions = gpd.GeoDataFrame(predictions, geometry='geometry') ground_df['geometry'] = ground_df.apply( lambda x: shapely.geometry.box(x.xmin, x.ymin, x.xmax, x.ymax), axis=1) ground_df = gpd.GeoDataFrame(ground_df, geometry='geometry') # match result = IoU.compute_IoU(ground_df, predictions) #add the label classes result["predicted_label"] = result.prediction_id.apply( lambda x: predictions.label.loc[x] if pd.notnull(x) else x) result["true_label"] = result.truth_id.apply( lambda x: ground_df.label.loc[x]) if savedir: image = np.array(Image.open("{}/{}".format(root_dir, plot_name)))[:, :, ::-1] image = visualize.plot_predictions(image, df=predictions) image = visualize.plot_predictions(image, df=ground_df, color=(0, 165, 255)) cv2.imwrite("{}/{}".format(savedir, plot_name), image) return result
def predict_image(model, image, return_plot, device, iou_threshold=0.1, color=None, thickness=1): """Predict an image with a deepforest model Args: image: a numpy array of a RGB image ranged from 0-255 path: optional path to read image from disk instead of passing image arg return_plot: Return image with plotted detections device: pytorch device of 'cuda' or 'cpu' for gpu prediction. Set internally. color: color of the bounding box as a tuple of BGR color, e.g. orange annotations is (0, 165, 255) thickness: thickness of the rectangle border line in px Returns: boxes: A pandas dataframe of predictions (Default) img: The input with predictions overlaid (Optional) """ if image.dtype != "float32": warnings.warn( f"Image type is {image.dtype}, transforming to float32. This assumes that the range of pixel values is 0-255, as opposed to 0-1.To suppress this warning, transform image (image.astype('float32')" ) image = image.astype("float32") image = preprocess.preprocess_image(image, device=device) with torch.no_grad(): prediction = model(image) # return None for no predictions if len(prediction[0]["boxes"]) == 0: return None # This function on takes in a single image. df = visualize.format_boxes(prediction[0]) df = across_class_nms(df, iou_threshold=iou_threshold) if return_plot: #Bring to gpu if not device.type == "cpu": image = image.cpu() # Cv2 likes no batch dim, BGR image and channels last, 0-255 image = np.array(image.squeeze(0)) image = np.rollaxis(image, 0, 3) image = image[:, :, ::-1] * 255 image = image.astype("uint8") image = visualize.plot_predictions(image, df, color=color, thickness=thickness) return image else: return df
def test_plot_predictions(m, tmpdir, label): ds = m.val_dataloader() batch = next(iter(ds)) paths, images, targets = batch for path, image, target in zip(paths, images, targets): target_df = visualize.format_boxes(target, scores=False) target_df["image_path"] = path image = np.array(image)[:, :, ::-1] image = np.rollaxis(image, 0, 3) target_df.label = label image = visualize.plot_predictions(image, target_df) assert image.dtype == "uint8"
def predict_file(model, csv_file, root_dir, savedir, device, iou_threshold=0.1, color=(0, 165, 255), thickness=1): """Create a dataset and predict entire annotation file Csv file format is .csv file with the columns "image_path", "xmin","ymin","xmax","ymax" for the image name and bounding box position. Image_path is the relative filename, not absolute path, which is in the root_dir directory. One bounding box per line. If "label" column is present, these are assumed to be annotations and will be plotted in a different color than predictions Args: csv_file: path to csv file root_dir: directory of images. If none, uses "image_dir" in config savedir: Optional. Directory to save image plots. device: pytorch device of 'cuda' or 'cpu' for gpu prediction. Set internally. color: color of the bounding box as a tuple of BGR color, e.g. orange annotations is (0, 165, 255) thickness: thickness of the rectangle border line in px Returns: df: pandas dataframe with bounding boxes, label and scores for each image in the csv file """ model.eval() df = pd.read_csv(csv_file) #Dataloader (when not shuffled) returns a tensor for each image in order paths = df.image_path.unique() ds = dataset.TreeDataset(csv_file=csv_file, root_dir=root_dir, transforms=None, train=False) prediction_list = [] with torch.no_grad(): for i in ds: i = i.to(device) prediction = model(torch.unsqueeze(i, 0)) prediction_list.append(prediction) prediction_list = [item for sublist in prediction_list for item in sublist] results = [] for index, prediction in enumerate(prediction_list): #If there is more than one class, apply NMS Loop through images and apply cross prediction = visualize.format_boxes(prediction) if len(prediction.label.unique()) > 1: prediction = across_class_nms(prediction, iou_threshold=iou_threshold) if savedir: # Just predict the images, even though we have the annotations image = np.array(Image.open("{}/{}".format( root_dir, paths[index])))[:, :, ::-1] image = visualize.plot_predictions(image, prediction) #Plot annotations if they exist annotations = df[df.image_path == paths[index]] image = visualize.plot_predictions(image, annotations, color=color, thickness=thickness) cv2.imwrite( "{}/{}.png".format(savedir, os.path.splitext(paths[index])[0]), image) prediction["image_path"] = paths[index] results.append(prediction) results = pd.concat(results, ignore_index=True) return results
def predict_tile(model, device, raster_path=None, image=None, patch_size=400, patch_overlap=0.05, iou_threshold=0.15, return_plot=False, mosaic=True, use_soft_nms=False, sigma=0.5, thresh=0.001, color=None, thickness=1): """For images too large to input into the model, predict_tile cuts the image into overlapping windows, predicts trees on each window and reassambles into a single array. Args: model: pytorch model device: pytorch device of 'cuda' or 'cpu' for gpu prediction. Set internally. numeric_to_label_dict: dictionary in which keys are numeric integers and values are character labels raster_path: Path to image on disk image (array): Numpy image array in BGR channel order following openCV convention patch_size: patch size default400, patch_overlap: patch overlap default 0.15, iou_threshold: Minimum iou overlap among predictions between windows to be suppressed. Defaults to 0.14. Lower values suppress more boxes at edges. return_plot: Should the image be returned with the predictions drawn? mosaic: If true, return a single annotations dataframe. If false, return a list of windows and predictions use_soft_nms: whether to perform Gaussian Soft NMS or not, if false, default perform NMS. sigma: variance of Gaussian function used in Gaussian Soft NMS thresh: the score thresh used to filter bboxes after soft-nms performed Returns: boxes (array): if return_plot, an image. windows (list): if mosaic = False, a two item tuple for each window of patch_size with predictions Otherwise a numpy array of predicted bounding boxes, scores and labels """ if image is not None: pass else: # load raster as image image = rio.open(raster_path).read() image = np.moveaxis(image, 0, 2) # Compute sliding window index windows = preprocess.compute_windows(image, patch_size, patch_overlap) predicted_boxes = [] crops = [] for index, window in enumerate(tqdm(windows)): # crop window and predict crop = image[windows[index].indices()] crop = crop.astype('float32') # crop is RGB channel order, change to BGR? boxes = predict_image(model=model, image=crop, return_plot=False, device=device) if boxes is not None: if mosaic: # transform the coordinates to original system xmin, ymin, xmax, ymax = windows[index].getRect() boxes.xmin = boxes.xmin + xmin boxes.xmax = boxes.xmax + xmin boxes.ymin = boxes.ymin + ymin boxes.ymax = boxes.ymax + ymin else: crops.append(crop) predicted_boxes.append(boxes) if len(predicted_boxes) == 0: print("No predictions made, returning None") return None if mosaic: predicted_boxes = pd.concat(predicted_boxes) # Non-max supression for overlapping boxes among window if patch_overlap == 0: mosaic_df = predicted_boxes else: print( f"{predicted_boxes.shape[0]} predictions in overlapping windows, applying non-max supression" ) # move prediciton to tensor boxes = torch.tensor( predicted_boxes[["xmin", "ymin", "xmax", "ymax"]].values, dtype=torch.float32) scores = torch.tensor(predicted_boxes.score.values, dtype=torch.float32) labels = predicted_boxes.label.values if not use_soft_nms: # Performs non-maximum suppression (NMS) on the boxes according to # their intersection-over-union (IoU). bbox_left_idx = nms(boxes=boxes, scores=scores, iou_threshold=iou_threshold) else: # Performs soft non-maximum suppression (soft-NMS) on the boxes. bbox_left_idx = soft_nms(boxes=boxes, scores=scores, sigma=sigma, thresh=thresh) bbox_left_idx = bbox_left_idx.numpy() new_boxes, new_labels, new_scores = boxes[bbox_left_idx].type( torch.int), labels[bbox_left_idx], scores[bbox_left_idx] # Recreate box dataframe image_detections = np.concatenate([ new_boxes, np.expand_dims(new_labels, axis=1), np.expand_dims(new_scores, axis=1) ], axis=1) mosaic_df = pd.DataFrame( image_detections, columns=["xmin", "ymin", "xmax", "ymax", "label", "score"]) print( f"{mosaic_df.shape[0]} predictions kept after non-max suppression" ) if return_plot: # Draw predictions on BGR image = image[:, :, ::-1] image = visualize.plot_predictions(image, mosaic_df, color=color, thickness=thickness) # Mantain consistancy with predict_image return image else: return mosaic_df else: return list(zip(predicted_boxes, crops))