def generate_cost_map(image): contours, canopies_mask = segmentation.extract_canopy_contours(image) cost_map = np.full((np.size(image, 0), np.size(image, 1)), fill_value=0, dtype=np.uint8) cv2.drawContours(cost_map, contours, contourIdx=-1, color=255, thickness=-1) cv2.drawContours(cost_map, contours, contourIdx=-1, color=200, thickness=90) cost_map = np.minimum(canopies_mask, cost_map) cv2.drawContours(cost_map, contours, contourIdx=-1, color=120, thickness=20) external_ring = np.full((np.size(image, 0), np.size(image, 1)), fill_value=0, dtype=np.uint8) cv2.drawContours(external_ring, contours, contourIdx=-1, color=50, thickness=65) cost_map = np.maximum(cost_map, external_ring) cost_map = cost_map / 255.0 return cost_map
def __init__(self, grid_dim_x, grid_dim_y, translation, orientation, shear, sigma, image, n, m, pattern, std_normalized_tree_scores_threshold=0.6): self.init_grid_dim_x = grid_dim_x self.init_grid_dim_y = grid_dim_y self.init_translation_x = translation[0] self.init_translation_y = translation[1] self.init_orientation = orientation self.init_shear = shear self.init_sigma = sigma self.canopies_mask = segmentation.extract_canopy_contours(image)[1] self.n = n self.m = m self.pattern = pattern self.std_normalized_tree_scores_threshold = std_normalized_tree_scores_threshold self.steps = [] self.width = image.shape[1] self.height = image.shape[0]
def estimate_rows_orientation(image, search_step=0.5, min_distance_between_peaks=200, min_peak_width=50): _, contours_mask = segmentation.extract_canopy_contours(image) angles_to_scores = {} for correction_angle in np.arange(start=-90, stop=90, step=search_step): rotation_mat = cv2.getRotationMatrix2D( (image.shape[1] / 2, image.shape[0] / 2), correction_angle, scale=1.0) rotated_contours_mask = cv2.warpAffine( contours_mask, rotation_mat, (contours_mask.shape[1], contours_mask.shape[0])) column_sums_vector = np.sum(rotated_contours_mask, axis=0) minima_indices, _ = find_peaks(column_sums_vector * (-1), distance=min_distance_between_peaks, width=min_peak_width) minima_values = [column_sums_vector[index] for index in minima_indices] mean_minima = np.mean( minima_values) if len(minima_values) > 0 else 1e30 angles_to_scores[correction_angle] = mean_minima return [ key for key, value in sorted( angles_to_scores.iteritems(), key=lambda (k, v): v, reverse=False) ][0] * (-1)
def find_optimal_grid(image): cropped_image, _, _ = cv_utils.crop_region(image, x_center=image.shape[1] / 2, y_center=image.shape[0] / 2, x_pixels=2700, y_pixels=1700) points = [(541, 403), (2281, 1263)] _, contours_mask = segmentation.extract_canopy_contours(cropped_image) basic_grid = get_basic_grid(points[0], points[1], nodes_num_x=6, nodes_num_y=4) delta_x_init_values = set([-int(1.8**i) for i in range(3)] + [int(1.8**i) for i in range(3)]) delta_y_init_values = set([-int(1.8**i) for i in range(3)] + [int(1.8**i) for i in range(3)]) angle_init_values = set([-int(1.4**i) for i in range(3)] + [int(1.4**i) for i in range(3)]) # in degrees scale_init_values = np.linspace(start=0.8, stop=1.2, num=6) init_values_combinations = list( product(delta_x_init_values, delta_y_init_values, angle_init_values, scale_init_values)) def objective_aux(delta_x, delta_y, angle, scale): return objective(basic_grid, contours_mask, delta_x, delta_y, angle, scale) objective_values = utils.distribute_evenly_on_all_cores( objective_aux, init_values_combinations)
def get_grid_scores_array(full_grid_np, image, sigma): _, contours_mask = segmentation.extract_canopy_contours(image) full_grid_scores_np = np.empty(full_grid_np.shape) for i in range(full_grid_np.shape[0]): for j in range((full_grid_np.shape[1])): if np.any(np.isnan(full_grid_np[(i, j)])): full_grid_scores_np[(i, j)] = np.nan else: x, y = full_grid_np[(i, j)] full_grid_scores_np[(i, j)] = _trunk_point_score( contours_mask, x, y, sigma) return full_grid_scores_np
def __init__(self, grid_dim_x, grid_dim_y, translation, orientation, shear, sigma, image, n): self.init_grid_dim_x = grid_dim_x self.init_grid_dim_y = grid_dim_y self.init_translation_x = translation[0] self.init_translation_y = translation[1] self.init_orientation = orientation self.init_shear = shear self.init_sigma = sigma self.contours_mask = segmentation.extract_canopy_contours(image)[1] self.n = n self.width = image.shape[1] self.height = image.shape[0]
def refine_trunk_locations(image, trunk_coordinates_np, sigma, dim_x, dim_y, samples_along_axis=30, window_shift=50): _, contours_mask = segmentation.extract_canopy_contours(image) refined_trunk_locations_df = pd.DataFrame( index=range(trunk_coordinates_np.shape[0]), columns=range(trunk_coordinates_np.shape[1])) window_size = int(np.max([dim_x, dim_y]) * 1.1) circle_radius = int(sigma * 1.2) for i in range(trunk_coordinates_np.shape[0]): for j in range(trunk_coordinates_np.shape[1]): if np.any(np.isnan(trunk_coordinates_np[(i, j)])): continue x, y = trunk_coordinates_np[(i, j)] max_score = -np.inf best_x, best_y = None, None for candidate_x, candidate_y in itertools.product( np.round( np.linspace(x - window_shift, x + window_shift, num=samples_along_axis)), np.round( np.linspace(y - window_shift, y + window_shift, num=samples_along_axis))): canopy_patch, _, _ = cv_utils.crop_region( contours_mask, candidate_x, candidate_y, window_size, window_size) circle_mask = np.full(canopy_patch.shape, fill_value=0, dtype=np.uint8) circle_mask = cv2.circle(circle_mask, center=(canopy_patch.shape[1] / 2, canopy_patch.shape[0] / 2), radius=circle_radius, color=255, thickness=-1) score = np.sum( cv2.bitwise_and(canopy_patch, canopy_patch, mask=circle_mask)) if score > max_score: max_score = score best_x, best_y = candidate_x, candidate_y refined_trunk_locations_df.loc[i, j] = (best_x, best_y) return np.array(refined_trunk_locations_df)
def get_grid_scores_array(full_grid_np, image, sigma): _, canopies_mask = segmentation.extract_canopy_contours(image) full_grid_scores_np = np.empty(full_grid_np.shape) full_grid_pose_to_score = {} for i in range(full_grid_np.shape[0]): for j in range((full_grid_np.shape[1])): if np.any(np.isnan(full_grid_np[(i, j)])): full_grid_scores_np[(i, j)] = np.nan else: x, y = full_grid_np[(i, j)] score = tree_score(canopies_mask, x, y, sigma)[1] full_grid_scores_np[(i, j)] = score full_grid_pose_to_score[(int(x), int(y))] = score return full_grid_scores_np, full_grid_pose_to_score
def generate_canopies_map(image, lower_color=None, upper_color=None, min_area=None): contours_map = np.full((np.size(image, 0), np.size(image, 1)), fill_value=0, dtype=np.uint8) contours, _ = segmentation.extract_canopy_contours(image, lower_color, upper_color, min_area) cv2.drawContours(contours_map, contours, contourIdx=-1, color=128, thickness=-1) cv2.drawContours(contours_map, contours, contourIdx=-1, color=255, thickness=3) return contours_map
def refine_trunk_locations(image, trunk_coordinates_np, sigma, dim_x, dim_y, samples_along_axis=14): _, canopies_mask = segmentation.extract_canopy_contours(image) refined_trunk_locations_df = pd.DataFrame( index=range(trunk_coordinates_np.shape[0]), columns=range(trunk_coordinates_np.shape[1])) window_size = int(np.max([dim_x, dim_y]) * 1.1) window_shift = int(sigma / 3) for i in range(trunk_coordinates_np.shape[0]): for j in range(trunk_coordinates_np.shape[1]): if np.any(np.isnan(trunk_coordinates_np[(i, j)])): continue x, y = trunk_coordinates_np[(i, j)] max_score = -np.inf best_x, best_y = None, None for candidate_x, candidate_y in itertools.product( np.round( np.linspace(x - window_shift, x + window_shift, num=samples_along_axis)), np.round( np.linspace(y - window_shift, y + window_shift, num=samples_along_axis))): canopy_patch, _, _ = cv_utils.crop_region( canopies_mask, candidate_x, candidate_y, window_size, window_size) score, _ = tree_score(canopy_patch, canopy_patch.shape[1] / 2, canopy_patch.shape[0] / 2, sigma) if score > max_score: max_score = score best_x, best_y = candidate_x, candidate_y refined_trunk_locations_df.loc[i, j] = (best_x, best_y) return np.array(refined_trunk_locations_df)
def find_tree_centroids(image, correction_angle): rotation_mat = cv2.getRotationMatrix2D( (image.shape[1] / 2, image.shape[0] / 2), correction_angle, scale=1.0) # TODO: verify order of coordinates rotated_image = cv2.warpAffine(image, rotation_mat, (image.shape[1], image.shape[0])) rotated_centroids = [] _, contours_mask = segmentation.extract_canopy_contours(rotated_image) column_sums_vector = np.sum(contours_mask, axis=0) aisle_centers, _ = find_peaks(column_sums_vector * (-1), distance=200, width=50) slices_and_cumsums = [] for tree_row_left_limit, tree_tow_right_limit in zip( aisle_centers[:-1], aisle_centers[1:]): tree_row = contours_mask[:, tree_row_left_limit:tree_tow_right_limit] row_sums_vector = np.sum(tree_row, axis=1) tree_locations_in_row, _ = find_peaks(row_sums_vector, distance=160, width=30) rotated_centroids.append([ (int(np.mean([tree_row_left_limit, tree_tow_right_limit])), tree_location) for tree_location in tree_locations_in_row ]) slices_and_cumsums.append((tree_row, row_sums_vector)) vertical_rows_centroids_np = np.float32( list(itertools.chain.from_iterable(rotated_centroids))).reshape( -1, 1, 2) rotation_mat = np.insert(cv2.getRotationMatrix2D( (image.shape[1] / 2, image.shape[0] / 2), correction_angle * (-1), scale=1.0), [2], [0, 0, 1], axis=0) # TODO: verify coordinates order centroids_np = cv2.perspectiveTransform(vertical_rows_centroids_np, rotation_mat) centroids = [tuple(elem) for elem in centroids_np[:, 0, :].tolist()] return centroids, rotated_centroids, aisle_centers, slices_and_cumsums
image = cv2.line(image, (int(center_of_mass[0]), int(center_of_mass[1])), (int(p2[0]), int(p2[1])), (255, 255, 0), 7, cv2.LINE_AA) return image if __name__ == '__main__': idx = 0 for image_path in image_paths_list: image = cv2.imread(image_path) # cropped_image = cv_utils.center_crop(image, 0.25, 0.25) cropped_image = cv_utils.crop_region(image, x_center=image.shape[1] / 2, y_center=image.shape[0] / 2, x_pixels=2700, y_pixels=1700) contours, contours_mask = canopy_contours.extract_canopy_contours( cropped_image) cv2.drawContours(cropped_image, contours, contourIdx=-1, color=(0, 255, 0), thickness=3) idx += 1 all_contours_points = np.concatenate([contour for contour in contours]) all_contours_points_2d_array = np.empty((len(all_contours_points), 2), dtype=np.float64) for i in range(all_contours_points_2d_array.shape[0]): all_contours_points_2d_array[i, 0] = all_contours_points[i, 0, 0] all_contours_points_2d_array[i, 1] = all_contours_points[i, 0, 1] cropped_image = get_orientation(all_contours_points_2d_array,
def task(self, **kwargs): verbose_mode = kwargs.get('verbose_mode') # Read params and data sources map_image_path = self.data_sources['map_image_path'] localization_image_path = self.data_sources['localization_image_path'] trajectory = self.data_sources['trajectory'] map_semantic_trunks = self.data_sources['map_semantic_trunks'] bounding_box_expand_ratio = self.params['bounding_box_expand_ratio'] roi_size = self.params['roi_size'] methods = self.params['methods'] downsample_rate = self.params['downsample_rate'] localization_resolution = self.params['localization_resolution'] use_canopies_masks = self.params['use_canopies_masks'] # Read images map_image = cv2.imread(map_image_path) localization_image = cv2.imread(localization_image_path) upper_left, lower_right = cv_utils.get_bounding_box( map_image, map_semantic_trunks.values(), expand_ratio=bounding_box_expand_ratio) map_image = map_image[upper_left[1]:lower_right[1], upper_left[0]:lower_right[0]] localization_image = localization_image[upper_left[1]:lower_right[1], upper_left[0]:lower_right[0]] if use_canopies_masks: _, map_image = segmentation.extract_canopy_contours(map_image) _, localization_image = segmentation.extract_canopy_contours( localization_image) cv2.imwrite(os.path.join(self.experiment_dir, 'map_image.jpg'), map_image) cv2.imwrite( os.path.join(self.experiment_dir, 'localization_image.jpg'), localization_image) # Initialize errors dataframe errors = pd.DataFrame(index=map( lambda point: '%s_%s' % (point[0], point[1]), trajectory), columns=methods) # Loop over points in trajectory for ugv_pose_idx, ugv_pose in enumerate(trajectory): if ugv_pose_idx % downsample_rate != 0: continue if ugv_pose_idx % MESSAGING_FREQUENCY == 0: _logger.info('At point #%d' % ugv_pose_idx) roi_image, _, _ = cv_utils.crop_region(localization_image, ugv_pose[0], ugv_pose[1], roi_size, roi_size) if verbose_mode: matches_image = map_image.copy() cv2.circle(matches_image, tuple(ugv_pose), radius=15, color=(0, 0, 255), thickness=-1) cv2.rectangle( matches_image, (ugv_pose[0] - roi_size / 2, ugv_pose[1] - roi_size / 2), (ugv_pose[0] + roi_size / 2, ugv_pose[1] + roi_size / 2), (0, 0, 255), thickness=2) for method in methods: matching_result = cv2.matchTemplate(map_image, roi_image, method=eval('cv2.%s' % method)) min_val, max_val, min_loc, max_loc = cv2.minMaxLoc( matching_result) if method in ['TM_SQDIFF', 'TM_SQDIFF_NORMED']: match_top_left = min_loc else: match_top_left = max_loc match_bottom_right = (match_top_left[0] + roi_image.shape[1], match_top_left[1] + roi_image.shape[0]) match_center = (match_top_left[0] + roi_image.shape[1] / 2, match_top_left[1] + roi_image.shape[0] / 2) error = np.sqrt((ugv_pose[0] - match_center[0])**2 + (ugv_pose[1] - match_center[1])**2) * localization_resolution errors.loc['%s_%s' % (ugv_pose[0], ugv_pose[1]), method] = error if verbose_mode: cv2.rectangle(matches_image, match_top_left, match_bottom_right, (255, 0, 0), thickness=2) cv2.circle(matches_image, match_center, radius=15, color=(255, 0, 0), thickness=-1) cv2.imwrite( os.path.join( self.repetition_dir, 'matches_%s_%s.jpg' % (ugv_pose[0], ugv_pose[1])), matches_image) # Save results errors.to_csv(os.path.join(self.experiment_dir, 'errors.csv'))
points_in_baseline=points_baseline, transformation_type='rigid') warped_obstacle_in_5_6_gray_image, _ = cv_utils.warp_image( image=obstacle_in_5_6_gray_image, points_in_image=points_obstacle_in_5_6, points_in_baseline=points_baseline, transformation_type='rigid') # _, baseline_to_obstacle_in_3_4_diff = compare_ssim(baseline_image, warped_obstacle_in_3_4_image, full=True) # baseline_to_obstacle_in_3_4_diff = (baseline_to_obstacle_in_3_4_diff * 255).astype('uint8') # _, baseline_to_obstacle_in_4_5_diff = compare_ssim(baseline_image, warped_obstacle_in_4_5_image, full=True) # baseline_to_obstacle_in_4_5_diff = (baseline_to_obstacle_in_4_5_diff * 255).astype('uint8') # _, baseline_to_obstacle_in_5_6_diff = compare_ssim(baseline_image, warped_obstacle_in_5_6_image, full=True) # baseline_to_obstacle_in_5_6_diff = (baseline_to_obstacle_in_5_6_diff * 255).astype('uint8') _, baseline_contours_mask = segmentation.extract_canopy_contours( baseline_image, margin_width=15, margin_color=255) baseline_contours_mask = cv2.dilate(baseline_contours_mask, kernel=np.ones((20, 20), np.uint8), iterations=1) baseline_contours_mask = 1 - (1 / 255.0) * baseline_contours_mask _, obstacle_in_3_4_contours_mask = segmentation.extract_canopy_contours( warped_obstacle_in_3_4_image, margin_width=15, margin_color=255) obstacle_in_3_4_contours_mask = cv2.dilate(obstacle_in_3_4_contours_mask, kernel=np.ones((20, 20), np.uint8), iterations=1) obstacle_in_3_4_contours_mask = 1 - (1 / 255.0) * obstacle_in_3_4_contours_mask _, obstacle_in_4_5_contours_mask = segmentation.extract_canopy_contours( warped_obstacle_in_4_5_image, margin_width=15, margin_color=255) obstacle_in_4_5_contours_mask = cv2.dilate(obstacle_in_4_5_contours_mask,
def task(self, **kwargs): image = cv2.imread(self.data_sources['map_image_path']) waypoints = self.data_sources['waypoints'] upper_left = self.data_sources['map_upper_left'] lower_right = self.data_sources['map_lower_right'] # Crop the image cropped_image = image[upper_left[1]:lower_right[1], upper_left[0]:lower_right[0]] waypoints = (np.array(waypoints) - np.array(upper_left)).tolist() # Get cost map cost_map = maps_generation.generate_cost_map(cropped_image) cv2.imwrite(os.path.join(self.repetition_dir, 'cost_map.jpg'), 255.0 * cost_map) # Plan a path path_planner = AstarPathPlanner(cost_map) trajectory = [] for section_start, section_end in zip(waypoints[:-1], waypoints[1:]): trajectory += list( path_planner.astar(tuple(section_start), tuple(section_end))) # Save results self.results[self.repetition_id]['trajectory'] = trajectory trajectory_on_cost_map_image = cv2.cvtColor(np.uint8(255.0 * cost_map), cv2.COLOR_GRAY2BGR) trajectory_on_cost_map_image = cv_utils.draw_points_on_image( trajectory_on_cost_map_image, trajectory, color=(0, 255, 255), radius=5) cv2.imwrite( os.path.join(self.repetition_dir, 'trajectory_on_cost_map.jpg'), trajectory_on_cost_map_image) self.results[ self.repetition_id]['trajectory_on_cost_map_path'] = os.path.join( self.repetition_dir, 'trajectory_on_cost_map.jpg') _, trajectory_on_mask_image = segmentation.extract_canopy_contours( cropped_image) trajectory_on_mask_image = cv2.cvtColor(trajectory_on_mask_image, cv2.COLOR_GRAY2BGR) trajectory_on_mask_image = cv_utils.draw_points_on_image( trajectory_on_mask_image, trajectory, color=(0, 255, 255), radius=5) cv2.imwrite( os.path.join(self.repetition_dir, 'trajectory_on_mask.jpg'), trajectory_on_mask_image) self.results[ self.repetition_id]['trajectory_on_mask_path'] = os.path.join( self.repetition_dir, 'trajectory_on_mask.jpg') trajectory_on_image = cv_utils.draw_points_on_image(cropped_image, trajectory, color=(0, 255, 255), radius=5) cv2.imwrite( os.path.join(self.repetition_dir, 'trajectory_on_image.jpg'), trajectory_on_image) self.results[ self.repetition_id]['trajectory_on_image_path'] = os.path.join( self.repetition_dir, 'trajectory_on_image.jpg')
if setup == 'apr': from content.data_pointers.lavi_april_18.dji import trunks_detection_results_dir as td_results_dir from content.data_pointers.lavi_april_18.dji import selected_trunks_detection_experiments as selected_td_experiments elif setup == 'nov1': from content.data_pointers.lavi_november_18.dji import trunks_detection_results_dir as td_results_dir from content.data_pointers.lavi_november_18.dji import plot1_selected_trunks_detection_experiments as selected_td_experiments else: raise NotImplementedError if __name__ == '__main__': execution_dir = utils.create_new_execution_folder('canopy_contours_drawer') with open(os.path.join(td_results_dir, selected_td_experiments[source_image_index], 'experiment_summary.json')) as f: td_summary = json.load(f) image = cv2.imread(td_summary['data_sources']) contours, canopies_mask = segmentation.extract_canopy_contours(image, min_area=min_area) image_with_contours = image.copy() cv2.drawContours(image_with_contours, contours, contourIdx=-1, color=(0, 255, 0), thickness=5) canopies_mask_with_contours = cv2.cvtColor(canopies_mask.copy(), cv2.COLOR_GRAY2BGR) cv2.drawContours(canopies_mask_with_contours, contours, contourIdx=-1, color=(0, 255, 0), thickness=5) canopies_mask_with_trunks = cv2.cvtColor(canopies_mask.copy(), cv2.COLOR_GRAY2BGR) canopies_mask_with_trunks = cv_utils.draw_points_on_image(canopies_mask_with_trunks, td_summary['results']['1']['semantic_trunks'].values(), color=(0, 220, 0)) canopies_mask_with_labeled_trunks = canopies_mask_with_trunks.copy() for trunk_label, trunk_pose in td_summary['results']['1']['semantic_trunks'].items(): canopies_mask_with_labeled_trunks = cv_utils.put_shaded_text_on_image(canopies_mask_with_labeled_trunks, label=trunk_label, location=trunk_pose, color=(0, 220, 0), offset=(15, 15)) cv2.imwrite(os.path.join(execution_dir, 'image.jpg'), image)
apr_noon_trunks.values(), transformation_type='affine') nov_image, _ = cv_utils.warp_image(nov_image, nov_trunks.values(), apr_noon_trunks.values(), transformation_type='affine') cv2.imwrite(os.path.join(execution_dir, 'apr_noon.jpg'), apr_noon_image) cv2.imwrite(os.path.join(execution_dir, 'apr_late_noon.jpg'), apr_late_noon_image) cv2.imwrite(os.path.join(execution_dir, 'apr_afternoon.jpg'), apr_afternoon_image) cv2.imwrite(os.path.join(execution_dir, 'apr_late_afternoon.jpg'), apr_late_afternoon_image) cv2.imwrite(os.path.join(execution_dir, 'nov.jpg'), nov_image) apr_noon_contours, _ = segmentation.extract_canopy_contours(apr_noon_image) apr_late_noon_contours, _ = segmentation.extract_canopy_contours( apr_late_noon_image) apr_afternoon_contours, _ = segmentation.extract_canopy_contours( apr_afternoon_image) apr_late_afternoon_contours, _ = segmentation.extract_canopy_contours( apr_late_afternoon_image) nov_contours, _ = segmentation.extract_canopy_contours(nov_image) cv2.drawContours(apr_noon_image, apr_noon_contours, contourIdx=-1, color=(0, 255, 0), thickness=3) cv2.drawContours(apr_late_noon_image, apr_late_noon_contours,
# Estimate sigma to one third of intra-row distance sigma = grid_dim_y / 3 # Get a grid of gaussians grid = trunks_detection_old_cv.get_grid(grid_dim_x, grid_dim_y, translation, orientation, shear, n=N) gaussians_filter = trunks_detection_old_cv.get_gaussians_grid_image( grid, sigma, cropped_image.shape[1], cropped_image.shape[0]) if viz_mode: viz_utils.show_image('gaussians', gaussians_filter) _, contours_mask = segmentation.extract_canopy_contours( cropped_image) filter_result = np.multiply(gaussians_filter, contours_mask) viz_utils.show_image('filter result', filter_result) # OPTIMIZATION TODO: improve and arrange opt = trunks_detection_old_cv._TrunksGridOptimization(grid_dim_x, grid_dim_y, translation, orientation, shear, sigma, cropped_image, n=N) nm = NelderMead(opt.target, opt.get_params()) optimized_grid_args, _ = nm.maximize(n_iter=30) optimized_grid_dim_x, optimized_grid_dim_y, optimized_translation_x, optimized_translation_y, optimized_orientation, optimized_shear, optimized_sigma = optimized_grid_args
def task(self, **kwargs): viz_mode = kwargs.get('viz_mode') verbose_mode = kwargs.get('verbose') # Read image image = cv2.imread(self.data_sources) cv2.imwrite(os.path.join(self.repetition_dir, 'image.jpg'), image) if viz_mode: viz_utils.show_image('image', image) # Save contours mask _, canopies_mask = segmentation.extract_canopy_contours(image) cv2.imwrite(os.path.join(self.repetition_dir, 'canopies_mask.jpg'), canopies_mask) # Crop central ROI cropped_image_size = int(np.min([image.shape[0], image.shape[1]]) * self.params['crop_ratio']) cropped_image, crop_origin, _ = cv_utils.crop_region(image, x_center=image.shape[1] / 2, y_center=image.shape[0] / 2, x_pixels=cropped_image_size, y_pixels=cropped_image_size) _, cropped_canopies_mask = segmentation.extract_canopy_contours(cropped_image) crop_square_image = image.copy() cv2.rectangle(crop_square_image, crop_origin, (crop_origin[0] + cropped_image_size, crop_origin[1] + cropped_image_size), color=(120, 0, 0), thickness=20) cv2.imwrite(os.path.join(self.repetition_dir, 'crop_square_image.jpg'), crop_square_image) cv2.imwrite(os.path.join(self.repetition_dir, 'cropped_image.jpg'), cropped_image) if viz_mode: viz_utils.show_image('cropped image', cropped_image) # Estimate orchard orientation orientation, angle_to_minima_mean, angle_to_sum_vector = trunks_detection.estimate_rows_orientation(cropped_image) rotation_mat = cv2.getRotationMatrix2D((cropped_image.shape[1] / 2, cropped_image.shape[0] / 2), orientation * (-1), scale=1.0) vertical_rows_image = cv2.warpAffine(cropped_image, rotation_mat, (cropped_image.shape[1], cropped_image.shape[0])) cv2.imwrite(os.path.join(self.repetition_dir, 'vertical_rows.jpg'), vertical_rows_image) if verbose_mode: angle_to_minima_mean_df = pd.DataFrame(angle_to_minima_mean.values(), index=angle_to_minima_mean.keys(), columns=['minima_mean']).sort_index() angle_to_minima_mean_df.to_csv(os.path.join(self.repetition_dir, 'angle_to_minima_mean.csv')) self.results[self.repetition_id]['angle_to_minima_mean_path'] = os.path.join(self.repetition_dir, 'angle_to_minima_mean.csv') max_sum_value = max(map(lambda vector: vector.max(), angle_to_sum_vector.values())) os.mkdir(os.path.join(self.repetition_dir, 'orientation_estimation')) for angle in angle_to_sum_vector: plt.figure() plt.plot(angle_to_sum_vector[angle], color='green') plt.xlabel('x') plt.ylabel('column sums') plt.ylim([(-0.05 * max_sum_value), int(max_sum_value * 1.05)]) plt.ticklabel_format(axis='y', style='sci', scilimits=(0, 4)) plt.autoscale(enable=True, axis='x', tight=True) plt.tight_layout() plt.savefig(os.path.join(self.repetition_dir, 'orientation_estimation', 'sums_vector_%.2f[deg].jpg' % angle)) rotation_mat = cv2.getRotationMatrix2D((cropped_canopies_mask.shape[1] / 2, cropped_canopies_mask.shape[0] / 2), angle, scale=1.0) rotated_canopies_mask = cv2.warpAffine(cropped_canopies_mask, rotation_mat, (cropped_canopies_mask.shape[1], cropped_canopies_mask.shape[0])) cv2.imwrite(os.path.join(self.repetition_dir, 'orientation_estimation', 'rotated_canopies_mask_%.2f[deg]_minima_mean=%.2f.jpg' % (angle, angle_to_minima_mean[angle])), rotated_canopies_mask) if viz_mode: viz_utils.show_image('vertical rows', vertical_rows_image) # Get tree centroids centroids, rotated_centroids, aisle_centers, slices_sum_vectors_and_trees, column_sums_vector = trunks_detection.find_tree_centroids(cropped_image, correction_angle=orientation * (-1)) _, vertical_rows_canopies_mask = segmentation.extract_canopy_contours(vertical_rows_image) vertical_rows_aisle_centers_image = cv_utils.draw_lines_on_image(cv2.cvtColor(vertical_rows_canopies_mask, cv2.COLOR_GRAY2BGR), lines_list=[((center, 0), (center, vertical_rows_image.shape[0])) for center in aisle_centers], color=(0, 0, 255)) slice_image, slice_row_sums_vector, tree_locations_in_row = slices_sum_vectors_and_trees[len(slices_sum_vectors_and_trees) / 2] tree_locations = [(slice_image.shape[1] / 2, vertical_location) for vertical_location in tree_locations_in_row] slice_image = cv_utils.draw_points_on_image(cv2.cvtColor(slice_image, cv2.COLOR_GRAY2BGR), tree_locations, color=(0, 0, 255)) cv2.imwrite(os.path.join(self.repetition_dir, 'vertical_rows_aisle_centers.jpg'), vertical_rows_aisle_centers_image) plt.figure() plt.plot(column_sums_vector, color='green') plt.xlabel('x') plt.ylabel('column sums') plt.ticklabel_format(axis='y', style='sci', scilimits=(0, 4)) plt.autoscale(enable=True, axis='x', tight=True) plt.tight_layout() plt.savefig(os.path.join(self.repetition_dir, 'vertical_rows_column_sums.jpg')) cv2.imwrite(os.path.join(self.repetition_dir, 'vertical_row_slice.jpg'), slice_image) plt.figure(figsize=(4, 5)) plt.plot(slice_row_sums_vector[::-1], range(len(slice_row_sums_vector)), color='green') plt.xlabel('row sums') plt.ylabel('y') plt.axes().set_aspect(60) plt.ticklabel_format(axis='x', style='sci', scilimits=(0, 4)) plt.autoscale(enable=True, axis='y', tight=True) plt.tight_layout() plt.savefig(os.path.join(self.repetition_dir, 'slice_row_sums.jpg')) vertical_rows_centroids_image = cv_utils.draw_points_on_image(vertical_rows_image, itertools.chain.from_iterable(rotated_centroids), color=(0, 0, 255)) cv2.imwrite(os.path.join(self.repetition_dir, 'vertical_rows_centroids.jpg'), vertical_rows_centroids_image) centroids_image = cv_utils.draw_points_on_image(cropped_image, centroids, color=(0, 0, 255)) cv2.imwrite(os.path.join(self.repetition_dir, 'centroids.jpg'), centroids_image) if viz_mode: viz_utils.show_image('vertical rows aisle centers', vertical_rows_aisle_centers_image) viz_utils.show_image('vertical rows centroids', vertical_rows_centroids_image) # Estimate grid parameters grid_dim_x, grid_dim_y = trunks_detection.estimate_grid_dimensions(rotated_centroids) shear, drift_vectors, drift_vectors_filtered = trunks_detection.estimate_shear(rotated_centroids) drift_vectors_image = cv_utils.draw_lines_on_image(vertical_rows_centroids_image, drift_vectors, color=(255, 255, 0), arrowed=True) cv2.imwrite(os.path.join(self.repetition_dir, 'drift_vectors.jpg'), drift_vectors_image) drift_vectors_filtered_image = cv_utils.draw_lines_on_image(vertical_rows_centroids_image, drift_vectors_filtered, color=(255, 255, 0), arrowed=True) cv2.imwrite(os.path.join(self.repetition_dir, 'drift_vectors_filtered.jpg'), drift_vectors_filtered_image) if viz_mode: viz_utils.show_image('drift vectors', drift_vectors_filtered_image) # Get essential grid essential_grid = trunks_detection.get_essential_grid(grid_dim_x, grid_dim_y, shear, orientation, n=self.params['grid_size_for_optimization']) essential_grid_shape = np.max(essential_grid, axis=0) - np.min(essential_grid, axis=0) margin = essential_grid_shape * 0.2 essential_grid_shifted = [tuple(elem) for elem in np.array(essential_grid) - np.min(essential_grid, axis=0) + margin / 2] estimated_grid_image = np.full((int(essential_grid_shape[1] + margin[1]), int(essential_grid_shape[0] + margin[0]), 3), 255, dtype=np.uint8) estimated_grid_image = cv_utils.draw_points_on_image(estimated_grid_image, essential_grid_shifted, color=(255, 90, 0), radius=25) cv2.imwrite(os.path.join(self.repetition_dir, 'estimated_grid.png'), estimated_grid_image) if viz_mode: viz_utils.show_image('estimated grid', estimated_grid_image) # Find translation of the grid positioned_grid, translation, drift_vectors = trunks_detection.find_min_mse_position(centroids, essential_grid, cropped_image.shape[1], cropped_image.shape[0]) if positioned_grid is None: raise ExperimentFailure positioned_grid_image = cv_utils.draw_points_on_image(cropped_image, positioned_grid, color=(255, 90, 0), radius=25) cv2.imwrite(os.path.join(self.repetition_dir, 'positioned_grid_only.jpg'), positioned_grid_image) positioned_grid_image = cv_utils.draw_points_on_image(positioned_grid_image, centroids, color=(0, 0, 255)) positioned_grid_image = cv_utils.draw_lines_on_image(positioned_grid_image, drift_vectors, color=(255, 255, 0), thickness=3) cv2.imwrite(os.path.join(self.repetition_dir, 'positioned_grid.jpg'), positioned_grid_image) if viz_mode: viz_utils.show_image('positioned grid', positioned_grid_image) # Estimate sigma as a portion of intra-row distance sigma = grid_dim_y * self.params['initial_sigma_to_dim_y_ratio'] # Get a grid of gaussians grid = trunks_detection.get_grid(grid_dim_x, grid_dim_y, translation, orientation, shear, n=self.params['grid_size_for_optimization']) gaussians_filter = trunks_detection.get_gaussians_grid_image(grid, sigma, cropped_image.shape[1], cropped_image.shape[0]) cv2.imwrite(os.path.join(self.repetition_dir, 'gaussians_filter.jpg'), 255.0 * gaussians_filter) filter_output = np.multiply(gaussians_filter, cropped_canopies_mask) cv2.imwrite(os.path.join(self.repetition_dir, 'filter_output.jpg'), filter_output) if viz_mode: viz_utils.show_image('gaussians filter', gaussians_filter) viz_utils.show_image('filter output', filter_output) # Optimize the squared grid optimized_grid, optimized_grid_args, optimization_steps = trunks_detection.optimize_grid(grid_dim_x, grid_dim_y, translation, orientation, shear, sigma, cropped_image, pattern=np.ones([self.params['grid_size_for_optimization'],self.params['grid_size_for_optimization']])) optimized_grid_dim_x, optimized_grid_dim_y, optimized_translation_x, optimized_translation_y, optimized_orientation, optimized_shear, optimized_sigma = optimized_grid_args self.results[self.repetition_id] = {'optimized_grid_dim_x': optimized_grid_dim_x, 'optimized_grid_dim_y': optimized_grid_dim_y, 'optimized_translation_x': optimized_translation_x, 'optimized_translation_y': optimized_translation_y, 'optimized_orientation': optimized_orientation, 'optimized_shear': optimized_shear, 'optimized_sigma': optimized_sigma} optimized_grid_image = cv_utils.draw_points_on_image(cropped_image, optimized_grid, color=(0, 255, 0)) optimized_grid_image = cv_utils.draw_points_on_image(optimized_grid_image, positioned_grid, color=(255, 90, 0)) cv2.imwrite(os.path.join(self.repetition_dir, 'optimized_square_grid.jpg'), optimized_grid_image) if verbose_mode: os.mkdir(os.path.join(self.repetition_dir, 'nelder_mead_steps')) self.results[self.repetition_id]['optimization_steps_scores'] = {} for step_idx, (step_grid, step_score, step_sigma) in enumerate(optimization_steps): self.results[self.repetition_id]['optimization_steps_scores'][step_idx] = step_score step_image = cropped_image.copy() step_gaussians_filter = trunks_detection.get_gaussians_grid_image(step_grid, step_sigma, cropped_image.shape[1], cropped_image.shape[0]) step_gaussians_filter = cv2.cvtColor((255.0 * step_gaussians_filter).astype(np.uint8), cv2.COLOR_GRAY2BGR) alpha = 0.5 weighted = cv2.addWeighted(step_image, alpha, step_gaussians_filter, 1 - alpha, gamma=0) update_indices = np.where(step_gaussians_filter != 0) step_image[update_indices] = weighted[update_indices] step_image = cv_utils.draw_points_on_image(step_image, step_grid, color=(0, 255, 0)) cv2.imwrite(os.path.join(self.repetition_dir, 'nelder_mead_steps', 'optimization_step_%d_[%.2f].jpg' % (step_idx, step_score)), step_image) if viz_mode: viz_utils.show_image('optimized square grid', optimized_grid_image) # Extrapolate full grid on the entire image full_grid_np = trunks_detection.extrapolate_full_grid(optimized_grid_dim_x, optimized_grid_dim_y, optimized_orientation, optimized_shear, base_grid_origin=np.array(optimized_grid[0]) + np.array(crop_origin), image_width=image.shape[1], image_height=image.shape[0]) full_grid_image = cv_utils.draw_points_on_image(image, [elem for elem in full_grid_np.flatten() if type(elem) is tuple], color=(0, 255, 0)) cv2.imwrite(os.path.join(self.repetition_dir, 'full_grid.jpg'), full_grid_image) if viz_mode: viz_utils.show_image('full grid', full_grid_image) # Match given orchard pattern to grid full_grid_scores_np, full_grid_pose_to_score = trunks_detection.get_grid_scores_array(full_grid_np, image, sigma) full_grid_with_scores_image = full_grid_image.copy() top_bottom_margin_size = int(0.05 * full_grid_with_scores_image.shape[0]) left_right_marign_size = int(0.05 * full_grid_with_scores_image.shape[1]) full_grid_with_scores_image = cv2.copyMakeBorder(full_grid_with_scores_image, top_bottom_margin_size, top_bottom_margin_size, left_right_marign_size, left_right_marign_size, cv2.BORDER_CONSTANT, dst=None, value=(255, 255, 255)) for pose, score in full_grid_pose_to_score.items(): pose = tuple(np.array(pose) + np.array([left_right_marign_size, top_bottom_margin_size])) full_grid_with_scores_image = cv_utils.put_shaded_text_on_image(full_grid_with_scores_image, '%.2f' % score, pose, color=(0, 255, 0), offset=(15, 15)) cv2.imwrite(os.path.join(self.repetition_dir, 'full_grid_with_scores.jpg'), full_grid_with_scores_image) orchard_pattern_np = self.params['orchard_pattern'] pattern_origin, origin_to_sub_scores_array = trunks_detection.fit_pattern_on_grid(full_grid_scores_np, orchard_pattern_np) if pattern_origin is None: raise ExperimentFailure if verbose_mode: os.mkdir(os.path.join(self.repetition_dir, 'pattern_matching')) for step_origin, step_sub_score_array in origin_to_sub_scores_array.items(): pattern_matching_image = image.copy() step_trunk_coordinates_np = full_grid_np[step_origin[0] : step_origin[0] + orchard_pattern_np.shape[0], step_origin[1] : step_origin[1] + orchard_pattern_np.shape[1]] step_trunk_points_list = step_trunk_coordinates_np.flatten().tolist() pattern_matching_image = cv_utils.draw_points_on_image(pattern_matching_image, step_trunk_points_list, color=(255, 255, 255), radius=25) for i in range(step_trunk_coordinates_np.shape[0]): for j in range(step_trunk_coordinates_np.shape[1]): step_trunk_coordinates = (int(step_trunk_coordinates_np[(i, j)][0]), int(step_trunk_coordinates_np[(i, j)][1])) pattern_matching_image = cv_utils.put_shaded_text_on_image(pattern_matching_image, '%.2f' % step_sub_score_array[(i, j)], step_trunk_coordinates, color=(255, 255, 255), offset=(20, 20)) pattern_matching_image = cv_utils.draw_points_on_image(pattern_matching_image, [elem for elem in full_grid_np.flatten() if type(elem) is tuple], color=(0, 255, 0)) mean_score = float(np.mean(step_sub_score_array)) cv2.imwrite(os.path.join(self.repetition_dir, 'pattern_matching', 'origin=%d_%d_score=%.2f.jpg' % (step_origin[0], step_origin[1], mean_score)), pattern_matching_image) trunk_coordinates_np = full_grid_np[pattern_origin[0] : pattern_origin[0] + orchard_pattern_np.shape[0], pattern_origin[1] : pattern_origin[1] + orchard_pattern_np.shape[1]] trunk_points_list = trunk_coordinates_np[orchard_pattern_np == 1] trunk_coordinates_orig_np = trunk_coordinates_np.copy() trunk_coordinates_np[orchard_pattern_np != 1] = np.nan semantic_trunks_image = cv_utils.draw_points_on_image(image, trunk_points_list, color=(255, 255, 255)) for i in range(trunk_coordinates_np.shape[0]): for j in range(trunk_coordinates_np.shape[1]): if np.any(np.isnan(trunk_coordinates_np[(i, j)])): continue trunk_coordinates = (int(trunk_coordinates_np[(i, j)][0]), int(trunk_coordinates_np[(i, j)][1])) tree_label = '%d/%s' % (j + 1, chr(65 + (trunk_coordinates_np.shape[0] - 1 - i))) semantic_trunks_image = cv_utils.put_shaded_text_on_image(semantic_trunks_image, tree_label, trunk_coordinates, color=(255, 255, 255), offset=(15, 15)) cv2.imwrite(os.path.join(self.repetition_dir, 'semantic_trunks.jpg'), semantic_trunks_image) if viz_mode: viz_utils.show_image('semantic trunks', semantic_trunks_image) # Refine trunk locations refined_trunk_coordinates_np = trunks_detection.refine_trunk_locations(image, trunk_coordinates_np, optimized_sigma, optimized_grid_dim_x, optimized_grid_dim_x) confidence = trunks_detection.get_trees_confidence(canopies_mask, refined_trunk_coordinates_np[orchard_pattern_np == 1], trunk_coordinates_orig_np[orchard_pattern_np == -1], optimized_sigma) refined_trunk_points_list = refined_trunk_coordinates_np[orchard_pattern_np == 1] refined_trunk_coordinates_np[orchard_pattern_np != 1] = np.nan refined_semantic_trunks_image = cv_utils.draw_points_on_image(image, refined_trunk_points_list, color=(255, 255, 255)) semantic_trunks = {} for i in range(refined_trunk_coordinates_np.shape[0]): for j in range(refined_trunk_coordinates_np.shape[1]): if np.any(np.isnan(refined_trunk_coordinates_np[(i, j)])): continue trunk_coordinates = (int(refined_trunk_coordinates_np[(i, j)][0]), int(refined_trunk_coordinates_np[(i, j)][1])) semantic_trunks['%d/%s' % (j + 1, chr(65 + (trunk_coordinates_np.shape[0] - 1 - i)))] = trunk_coordinates tree_label = '%d/%s' % (j + 1, chr(65 + (refined_trunk_coordinates_np.shape[0] - 1 - i))) refined_semantic_trunks_image = cv_utils.put_shaded_text_on_image(refined_semantic_trunks_image, tree_label, trunk_coordinates, color=(255, 255, 255), offset=(15, 15)) tree_scores_stats = trunks_detection.get_tree_scores_stats(canopies_mask, trunk_points_list, optimized_sigma) self.results[self.repetition_id]['semantic_trunks'] = semantic_trunks self.results[self.repetition_id]['tree_scores_stats'] = tree_scores_stats self.results[self.repetition_id]['confidence'] = confidence cv2.imwrite(os.path.join(self.repetition_dir, 'refined_semantic_trunks[%.2f].jpg' % confidence), refined_semantic_trunks_image) if viz_mode: viz_utils.show_image('refined semantic trunks', refined_semantic_trunks_image)
def task(self, **kwargs): # Read images image = cv2.imread(self.data_sources['image_path']) baseline_image = cv2.imread(self.data_sources['baseline_image_path']) _, baseline_canopies_mask = segmentation.extract_canopy_contours( baseline_image) cv2.imwrite(os.path.join(self.repetition_dir, 'image.jpg'), image) cv2.imwrite(os.path.join(self.repetition_dir, 'baseline_image.jpg'), baseline_image) # Align images by markers marker_locations = self.data_sources['markers_locations'] baseline_marker_locations = self.data_sources[ 'baseline_markers_locations'] warped_image_by_markers, _ = cv_utils.warp_image( image=image, points_in_image=marker_locations, points_in_baseline=baseline_marker_locations) cv2.imwrite(os.path.join(self.repetition_dir, 'warped by markers.jpg'), warped_image_by_markers) _, warped_canopies_mask_by_markers = segmentation.extract_canopy_contours( warped_image_by_markers) mse = cv_utils.calculate_image_similarity( baseline_canopies_mask, warped_canopies_mask_by_markers, method='mse') ssim = cv_utils.calculate_image_similarity( baseline_canopies_mask, warped_canopies_mask_by_markers, method='ssim') self.results['by_markers'] = {'mse': mse, 'ssim': ssim} # Align images by typical flow warped_image_by_orb, orb_matches_image = typical_image_alignment.orb_based_registration( image, baseline_image, transformation_type='affine') cv2.imwrite(os.path.join(self.repetition_dir, 'warped by orb.jpg'), warped_image_by_orb) cv2.imwrite(os.path.join(self.repetition_dir, 'orb matches.jpg'), orb_matches_image) _, warped_canopies_mask_by_orb = segmentation.extract_canopy_contours( warped_image_by_orb) mse = cv_utils.calculate_image_similarity(baseline_canopies_mask, warped_canopies_mask_by_orb, method='mse') ssim = cv_utils.calculate_image_similarity(baseline_canopies_mask, warped_canopies_mask_by_orb, method='ssim') self.results['by_orb'] = {'mse': mse, 'ssim': ssim} # Align images by trunks points trunks = self.data_sources['trunks_points'] baseline_trunks = self.data_sources['baseline_trunks_points'] warped_image_by_trunks, _ = cv_utils.warp_image( image=image, points_in_image=trunks, points_in_baseline=baseline_trunks) cv2.imwrite(os.path.join(self.repetition_dir, 'warped by trunks.jpg'), warped_image_by_trunks) _, warped_canopies_mask_by_trunks = segmentation.extract_canopy_contours( warped_image_by_trunks) mse = cv_utils.calculate_image_similarity( baseline_canopies_mask, warped_canopies_mask_by_trunks, method='mse') ssim = cv_utils.calculate_image_similarity( baseline_canopies_mask, warped_canopies_mask_by_trunks, method='ssim') self.results['by_trunks'] = {'mse': mse, 'ssim': ssim}
def task(self, **kwargs): viz_mode = kwargs.get('viz_mode') # Read image image = cv2.imread(self.data_sources) cv2.imwrite(os.path.join(self.repetition_dir, 'image.jpg'), image) if viz_mode: viz_utils.show_image('image', image) # Save contours mask _, contours_mask = segmentation.extract_canopy_contours(image) cv2.imwrite(os.path.join(self.repetition_dir, 'contours_mask.jpg'), contours_mask) # Crop central ROI cropped_image_size = np.min([image.shape[0], image.shape[1]]) * self.params['crop_ratio'] cropped_image, crop_origin, _ = cv_utils.crop_region(image, x_center=image.shape[1] / 2, y_center=image.shape[0] / 2, x_pixels=cropped_image_size, y_pixels=cropped_image_size) cv2.imwrite(os.path.join(self.repetition_dir, 'cropped_image.jpg'), cropped_image) if viz_mode: viz_utils.show_image('cropped image', cropped_image) # Estimate orchard orientation orientation = trunks_detection_old_cv.estimate_rows_orientation(cropped_image) rotation_mat = cv2.getRotationMatrix2D((cropped_image.shape[1] / 2, cropped_image.shape[0] / 2), orientation * (-1), scale=1.0) vertical_rows_image = cv2.warpAffine(cropped_image, rotation_mat, (cropped_image.shape[1], cropped_image.shape[0])) cv2.imwrite(os.path.join(self.repetition_dir, 'vertical_rows.jpg'), vertical_rows_image) if viz_mode: viz_utils.show_image('vertical rows', vertical_rows_image) # Get tree centroids centroids, rotated_centroids, aisle_centers, slices_and_cumsums = trunks_detection_old_cv.find_tree_centroids(cropped_image, correction_angle=orientation * (-1)) vertical_rows_aisle_centers_image = cv_utils.draw_lines_on_image(vertical_rows_image, lines_list=[((center, 0), (center, vertical_rows_image.shape[0])) for center in aisle_centers], color=(0, 0, 255)) cv2.imwrite(os.path.join(self.repetition_dir, 'vertical_rows_aisle_centers.jpg'), vertical_rows_aisle_centers_image) slice_image, cumsum_vector = slices_and_cumsums[len(slices_and_cumsums) / 2] cv2.imwrite(os.path.join(self.repetition_dir, 'vertical_row_slice.jpg'), slice_image) fig = plt.figure() plt.plot(cumsum_vector) plt.savefig(os.path.join(self.repetition_dir, 'cumsum_vector.jpg')) vertical_rows_centroids_image = cv_utils.draw_points_on_image(vertical_rows_image, itertools.chain.from_iterable(rotated_centroids), color=(0, 0, 255)) cv2.imwrite(os.path.join(self.repetition_dir, 'vertical_rows_centroids.jpg'), vertical_rows_centroids_image) if viz_mode: viz_utils.show_image('vertical rows aisle centers', vertical_rows_aisle_centers_image) viz_utils.show_image('vertical rows centroids', vertical_rows_centroids_image) # Estimate grid parameters grid_dim_x, grid_dim_y = trunks_detection_old_cv.estimate_grid_dimensions(rotated_centroids) shear, drift_vectors = trunks_detection_old_cv.estimate_shear(rotated_centroids) drift_vectors_image = cv_utils.draw_lines_on_image(vertical_rows_centroids_image, drift_vectors, color=(255, 255, 0)) cv2.imwrite(os.path.join(self.repetition_dir, 'drift_vectors.jpg'), drift_vectors_image) if viz_mode: viz_utils.show_image('drift vectors', drift_vectors_image) # Get essential grid essential_grid = trunks_detection_old_cv.get_essential_grid(grid_dim_x, grid_dim_y, shear, orientation, n=self.params['grid_size_for_optimization']) essential_grid_shape = np.max(essential_grid, axis=0) - np.min(essential_grid, axis=0) margin = essential_grid_shape * 0.2 essential_grid_shifted = [tuple(elem) for elem in np.array(essential_grid) - np.min(essential_grid, axis=0) + margin / 2] estimated_grid_image = np.full((int(essential_grid_shape[1] + margin[1]), int(essential_grid_shape[0] + margin[0]), 3), 0, dtype=np.uint8) estimated_grid_image = cv_utils.draw_points_on_image(estimated_grid_image, essential_grid_shifted, color=(255, 0, 0)) cv2.imwrite(os.path.join(self.repetition_dir, 'estimated_grid.jpg'), estimated_grid_image) if viz_mode: viz_utils.show_image('estimated grid', estimated_grid_image) # Find translation of the grid positioned_grid, translation, drift_vectors = trunks_detection_old_cv.find_min_mse_position(centroids, essential_grid, cropped_image.shape[1], cropped_image.shape[0]) if positioned_grid is None: raise ExperimentFailure positioned_grid_image = cv_utils.draw_points_on_image(cropped_image, positioned_grid, color=(255, 0, 0), radius=20) positioned_grid_image = cv_utils.draw_points_on_image(positioned_grid_image, centroids, color=(0, 0, 255), radius=10) positioned_grid_image = cv_utils.draw_lines_on_image(positioned_grid_image, drift_vectors, color=(255, 255, 0), thickness=3) cv2.imwrite(os.path.join(self.repetition_dir, 'positioned_grid.jpg'), positioned_grid_image) if viz_mode: viz_utils.show_image('positioned grid', positioned_grid_image) # Estimate sigma as a portion of intra-row distance sigma = grid_dim_y * self.params['initial_sigma_to_dim_y_ratio'] # Get a grid of gaussians grid = trunks_detection_old_cv.get_grid(grid_dim_x, grid_dim_y, translation, orientation, shear, n=self.params['grid_size_for_optimization']) gaussians_filter = trunks_detection_old_cv.get_gaussians_grid_image(grid, sigma, cropped_image.shape[1], cropped_image.shape[0]) cv2.imwrite(os.path.join(self.repetition_dir, 'gaussians_filter.jpg'), 255.0 * gaussians_filter) _, contours_mask = segmentation.extract_canopy_contours(cropped_image) filter_output = np.multiply(gaussians_filter, contours_mask) cv2.imwrite(os.path.join(self.repetition_dir, 'filter_output.jpg'), filter_output) if viz_mode: viz_utils.show_image('gaussians filter', gaussians_filter) viz_utils.show_image('filter output', filter_output) # Optimize the grid optimized_grid, optimized_grid_args = trunks_detection_old_cv.optimize_grid(grid_dim_x, grid_dim_y, translation, orientation, shear, sigma, cropped_image, n=self.params['grid_size_for_optimization']) optimized_grid_dim_x, optimized_grid_dim_y, optimized_translation_x, optimized_translation_y, optimized_orientation, optimized_shear, optimized_sigma = optimized_grid_args self.results[self.repetition_id] = {'optimized_grid_dim_x': optimized_grid_dim_x, 'optimized_grid_dim_y': optimized_grid_dim_y, 'optimized_translation_x': optimized_translation_x, 'optimized_translation_y': optimized_translation_y, 'optimized_orientation': optimized_orientation, 'optimized_shear': optimized_shear, 'optimized_sigma': optimized_sigma} optimized_grid_image = cv_utils.draw_points_on_image(cropped_image, optimized_grid, color=(0, 255, 0)) optimized_grid_image = cv_utils.draw_points_on_image(optimized_grid_image, positioned_grid, color=(255, 0, 0)) cv2.imwrite(os.path.join(self.repetition_dir, 'optimized_grid.jpg'), optimized_grid_image) if viz_mode: viz_utils.show_image('optimized grid', optimized_grid_image) # Extrapolate full grid on the entire image full_grid_np = trunks_detection_old_cv.extrapolate_full_grid(optimized_grid_dim_x, optimized_grid_dim_y, optimized_orientation, optimized_shear, base_grid_origin=np.array(optimized_grid[0]) + np.array(crop_origin), image_width=image.shape[1], image_height=image.shape[0]) full_grid_image = cv_utils.draw_points_on_image(image, [elem for elem in full_grid_np.flatten() if type(elem) is tuple], color=(255, 0, 0)) cv2.imwrite(os.path.join(self.repetition_dir, 'full_grid.jpg'), full_grid_image) if viz_mode: viz_utils.show_image('full grid', full_grid_image) # Match given orchard pattern to grid full_grid_scores_np = trunks_detection_old_cv.get_grid_scores_array(full_grid_np, image, optimized_sigma) orchard_pattern_np = self.params['orchard_pattern'] pattern_origin, pattern_match_score = trunks_detection_old_cv.fit_pattern_on_grid(full_grid_scores_np, orchard_pattern_np) if pattern_origin is None: raise ExperimentFailure self.results[self.repetition_id]['pattern_match_score'] = pattern_match_score trunk_coordinates_np = full_grid_np[pattern_origin[0] : pattern_origin[0] + orchard_pattern_np.shape[0], pattern_origin[1] : pattern_origin[1] + orchard_pattern_np.shape[1]] trunk_points_list = trunk_coordinates_np[orchard_pattern_np != -1] trunk_coordinates_np[orchard_pattern_np == -1] = np.nan semantic_trunks_image = cv_utils.draw_points_on_image(image, trunk_points_list, color=(255, 255, 255)) for i in range(trunk_coordinates_np.shape[0]): for j in range(trunk_coordinates_np.shape[1]): if np.any(np.isnan(trunk_coordinates_np[(i, j)])): continue label_coordinates = (int(trunk_coordinates_np[(i, j)][0]) + 15, int(trunk_coordinates_np[(i, j)][1]) + 15) tree_label = '%d/%s' % (j + 1, chr(65 + (trunk_coordinates_np.shape[0] - 1 - i))) cv2.putText(semantic_trunks_image, tree_label, label_coordinates, fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=2, color=(255, 255, 255), thickness=8, lineType=cv2.LINE_AA) cv2.imwrite(os.path.join(self.repetition_dir, 'semantic_trunks.jpg'), semantic_trunks_image) if viz_mode: viz_utils.show_image('semantic trunks', semantic_trunks_image) # Refine trunk locations refined_trunk_coordinates_np = trunks_detection_old_cv.refine_trunk_locations(image, trunk_coordinates_np, optimized_sigma, optimized_grid_dim_x, optimized_grid_dim_x) refined_trunk_points_list = refined_trunk_coordinates_np[orchard_pattern_np != -1] refined_trunk_coordinates_np[orchard_pattern_np == -1] = np.nan refined_semantic_trunks_image = cv_utils.draw_points_on_image(image, refined_trunk_points_list, color=(255, 255, 255)) semantic_trunks = {} for i in range(refined_trunk_coordinates_np.shape[0]): for j in range(refined_trunk_coordinates_np.shape[1]): if np.any(np.isnan(refined_trunk_coordinates_np[(i, j)])): continue trunk_coordinates = (int(refined_trunk_coordinates_np[(i, j)][0]), int(refined_trunk_coordinates_np[(i, j)][1])) label_coordinates = tuple(np.array(trunk_coordinates) + np.array([15, 15])) semantic_trunks['%d/%s' % (j + 1, chr(65 + (trunk_coordinates_np.shape[0] - 1 - i)))] = trunk_coordinates tree_label = '%d/%s' % (j + 1, chr(65 + (refined_trunk_coordinates_np.shape[0] - 1 - i))) cv2.putText(refined_semantic_trunks_image, tree_label, label_coordinates, fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=2, color=(255, 255, 255), thickness=8, lineType=cv2.LINE_AA) cv2.imwrite(os.path.join(self.repetition_dir, 'refined_semantic_trunks.jpg'), refined_semantic_trunks_image) self.results[self.repetition_id]['semantic_trunks'] = semantic_trunks if viz_mode: viz_utils.show_image('refined semantic trunks', refined_semantic_trunks_image)