def get_plot(hvf_image_gray, y_ratio, y_size, x_ratio, x_size, plot_type, icon_type): plot_image = Image_Utils.slice_image(hvf_image_gray, y_ratio, y_size, x_ratio, x_size) hvf_image_gray_process = Image_Utils.preprocess_image( hvf_image_gray.copy()) plot_image_process = Image_Utils.slice_image(hvf_image_gray_process, y_ratio, y_size, x_ratio, x_size) # Get bounding box from processed image: top_left, w, h = Hvf_Plot_Array.get_bounding_box(plot_image_process) bottom_right = (top_left[0] + w, top_left[1] + h) # Need to specifically handle raw value plot - can have a discontinuity in the # x axis (with triangle icon), which causes a mis-fit. So need to fill in x-axis # and try again cv2.line(plot_image_process, (top_left[0], top_left[1] + int(h / 2)), (top_left[0] + w, top_left[1] + int(h / 2)), (0), max(int(h * 0.015), 1)) top_left, w, h = Hvf_Plot_Array.get_bounding_box(plot_image_process) bottom_right = (top_left[0] + w, top_left[1] + h) # For debugging: Draw rectangle around the plot - MUST BE COMMENTED OUT, BECAUSE # IT WILL INTERFERE WITH LATER PLOT EXTRACTIONS #cv2.rectangle(plot_image, top_left, bottom_right, 0, 2) # Debug function for showing the plot: #show_plot_func = (lambda : cv2.imshow("Bound rect for plot " + plot_type, plot_image)) #Logger.get_logger().log_function(Logger.DEBUG_FLAG_DEBUG, show_plot_func); #cv2.waitKey(); # Slice out the axes plot on the original: tight_plot = plot_image[top_left[1]:(top_left[1] + h), top_left[0]:(top_left[0] + w)] # And extract the values from the array: plot_array = Hvf_Plot_Array.extract_values_from_plot( tight_plot, plot_type, icon_type) # Return the array: return plot_array, tight_plot
def is_pattern_not_shown(hvf_image_gray, y_ratio, y_size, x_ratio, x_size): # Calculate height/width for calculation later: height = np.size(hvf_image_gray, 0) width = np.size(hvf_image_gray, 1) # Slice image: hvf_image_gray = Image_Utils.preprocess_image(hvf_image_gray) sliced_img = Image_Utils.slice_image(hvf_image_gray, y_ratio, y_size, x_ratio, x_size) # Try to detect a bounding box: top_left, w, h = Hvf_Plot_Array.get_bounding_box(sliced_img) # Calculate the relative (percentage) size of the bounding box compared to slice: box_ratio_w = w / (x_size * width) box_ratio_h = h / (y_size * height) # Define a threshold below which if the size ratio is, we declare that the pattern # is not detected: threshold_size = 0.3 return (box_ratio_w < threshold_size or box_ratio_h < threshold_size)
def identify_digit(plot_element, allow_search_zero): # We template match against all icons and look for best fit: best_match = None best_val = None best_loc = None best_scale_factor = None height = np.size(plot_element, 0) width = np.size(plot_element, 1) # Can skip 0 if flag tells us to. This can help maximize accuracy in low-res cases # Do this when we know something about the digit (it is a leading digit, etc) start_index = 0 if not allow_search_zero: start_index = 1 for ii in range(start_index, len(Hvf_Value.value_icon_templates.keys())): for dir in Hvf_Value.value_icon_templates[ii]: # First, scale our template value: val_icon = Hvf_Value.value_icon_templates[ii][dir] plot_element_temp = plot_element.copy() scale_factor = 1 # Use the smaller factor to make sure we fit into the element icon if (height < np.size(val_icon, 0)): # Need to upscale plot_element scale_factor = np.size(val_icon, 0) / height plot_element_temp = cv2.resize(plot_element_temp, (0, 0), fx=scale_factor, fy=scale_factor) else: # Need to upscale val_icon scale_factor = height / (np.size(val_icon, 0)) val_icon = cv2.resize(val_icon, (0, 0), fx=scale_factor, fy=scale_factor) # In case the original is too small by width compared to value_icon, need # to widen - do so by copymakeborder replicate if (np.size(plot_element_temp, 1) < np.size(val_icon, 1)): border = np.size(val_icon, 1) - np.size( plot_element_temp, 1) #plot_element_temp = cv2.copyMakeBorder(plot_element_temp,0,0,0,border,cv2.BORDER_CONSTANT,0); # Apply template matching: temp_matching = cv2.matchTemplate(plot_element_temp, val_icon, cv2.TM_CCOEFF_NORMED) # Grab our result min_val, max_val, min_loc, max_loc = cv2.minMaxLoc( temp_matching) Logger.get_logger().log_msg( Logger.DEBUG_FLAG_DEBUG, "Matching against " + str(ii) + ": " + str(max_val)) # Check to see if this is our best fit yet: if (best_match is None or max_val > best_match): # This is best fit - record the match value and the actual value best_match = max_val best_val = ii best_loc = max_loc best_scale_factor = scale_factor # TODO: refine specific cases that tend to be misclassified # 1 vs 4 if (best_val == 4 or best_val == 1): # Cut number in half and take bottom half -> find contours # If width of contour is most of element --> 4 # otherwise, 1 bottom_half = Image_Utils.slice_image(plot_element, 0.5, 0.5, 0, 1) cnts, hierarchy = cv2.findContours( cv2.bitwise_not(bottom_half.copy()), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) # Sort contours by width largest_contour = sorted(cnts, key=Hvf_Value.contour_width, reverse=True)[0] if (Hvf_Value.contour_width(largest_contour) > width * 0.8): best_val = 4 else: best_val = 1 return best_val, best_loc, best_scale_factor, best_match
def extract_values_from_plot(plot_image, plot_type, icon_type): # First, image process for best readability: #plot_image = cv2.GaussianBlur(plot_image, (5,5), 0) plot_image_backup = plot_image.copy() # Perform image processing depending on plot type: if (icon_type == Hvf_Plot_Array.PLOT_PERC): plot_image = cv2.bitwise_not( cv2.adaptiveThreshold(plot_image, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 11, 5)) elif (icon_type == Hvf_Plot_Array.PLOT_VALUE): #plot_image = cv2.GaussianBlur(plot_image, (5,5), 0) ret2, plot_image = cv2.threshold( plot_image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) kernel_size = 31 mean_offset = 15 plot_image_backup = cv2.bitwise_not( cv2.adaptiveThreshold(plot_image_backup, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, kernel_size, mean_offset)) kernel = np.ones((3, 3), np.uint8) # For readability, grab our height/width: plot_width = np.size(plot_image, 1) plot_height = np.size(plot_image, 0) # The elements are laid out roughly within a 10x10 grid NUM_CELLS_ROW = Hvf_Plot_Array.NUM_OF_PLOT_ROWS NUM_CELLS_COL = Hvf_Plot_Array.NUM_OF_PLOT_COLS # Delete triangle icon, if we can find it: if (plot_type == Hvf_Plot_Array.PLOT_RAW): if not (Hvf_Plot_Array.find_and_delete_triangle_icon( plot_image, "v1")): Hvf_Plot_Array.find_and_delete_triangle_icon(plot_image, "v2") # Mask out corners: corner_mask = Hvf_Plot_Array.generate_corner_mask( plot_width, plot_height) plot_image = cv2.bitwise_or(plot_image, cv2.bitwise_not(corner_mask)) # First, declare our return value array, no need to really initialize bc we'll # iterate through it plot_values_array = 0 if (icon_type == Hvf_Plot_Array.PLOT_PERC): plot_values_array = np.zeros((NUM_CELLS_COL, NUM_CELLS_ROW), dtype=Hvf_Perc_Icon) elif (icon_type == Hvf_Plot_Array.PLOT_VALUE): plot_values_array = np.zeros((NUM_CELLS_COL, NUM_CELLS_ROW), dtype=Hvf_Value) plot_image = Hvf_Plot_Array.delete_plot_axes(plot_image) # Grab the grid lines: grid_line_dict = Hvf_Plot_Array.get_plot_grid_lines( plot_image, plot_type, icon_type) plot_image_debug_copy = plot_image.copy() # Debug code - draws out slicing for the elements on the plot: for c in range(Hvf_Plot_Array.NUM_OF_PLOT_COLS + 1): x = int(grid_line_dict['col_list'][c] * plot_width) #cv2.line(plot_image_debug_copy, (x, 0), (x, plot_height), (0), 1); for r in range(Hvf_Plot_Array.NUM_OF_PLOT_ROWS + 1): y = int(grid_line_dict['row_list'][r] * plot_height) #cv2.line(plot_image_debug_copy, (0, y), (plot_width, y), (0), 1); # Debug function for showing the plot: show_plot_func = ( lambda: cv2.imshow("plot " + icon_type, plot_image_debug_copy)) Logger.get_logger().log_function(Logger.DEBUG_FLAG_DEBUG, show_plot_func) #cv2.imshow("plot " + icon_type, plot_image_debug_copy) #cv2.waitKey(); # We iterate through our array, then slice out the appropriate cell from the plot for x in range(0, NUM_CELLS_COL): for y in range(0, NUM_CELLS_ROW): # Debug info for indicating what cell we're computing: Logger.get_logger().log_msg(Logger.DEBUG_FLAG_INFO, "Cell " + str(x) + "," + str(y)) # Grab our cell slice for the plot element # (remember arguments: slice_image(image, y_ratio, y_size, x_ratio, x_size): # The height of the axes tends to extend ~2.5% past the elements on top, bottom # The width of the axes tends to extend # So we take that into account when we take the slice row_grid_val = grid_line_dict['row_list'][y] row_grid_val_size = grid_line_dict['row_list'][ y + 1] - grid_line_dict['row_list'][y] col_grid_val = grid_line_dict['col_list'][x] col_grid_val_size = grid_line_dict['col_list'][ x + 1] - grid_line_dict['col_list'][x] cell_slice = Image_Utils.slice_image(plot_image, row_grid_val, row_grid_val_size, col_grid_val, col_grid_val_size) cell_slice_backup = Image_Utils.slice_image( plot_image_backup, row_grid_val, row_grid_val_size, col_grid_val, col_grid_val_size) cell_object = 0 # Then, need to analyze to figure out what element is in this position # What we look for depends on type of plot - perc vs value if (icon_type == Hvf_Plot_Array.PLOT_PERC): if (Hvf_Plot_Array.PLOT_ELEMENT_BOOLEAN_MASK[y][x]): # This element needs to be detected # Because this step relies on many things going right, possible that our # slices are not amenable to template matching and cause an error # So, try it under a try-except clause. If failure, we place a failure # placeholder try: cell_object = Hvf_Perc_Icon.get_perc_icon_from_image( cell_slice) Logger.get_logger().log_msg( Logger.DEBUG_FLAG_INFO, "Percentile Icon detected: " + cell_object.get_display_string()) except: Logger.get_logger().log_msg( Logger.DEBUG_FLAG_WARNING, "Cell " + str(x) + "," + str(y) + ": Percentile icon detection failure") cell_object = Hvf_Perc_Icon.get_perc_icon_from_char( Hvf_Perc_Icon.PERC_FAILURE_CHAR) raise Exception(str(e)) else: # This is a no-detect element, so just instantiate a blank: cell_object = Hvf_Perc_Icon.get_perc_icon_from_char( Hvf_Perc_Icon.PERC_NO_VALUE_CHAR) Logger.get_logger().log_msg( Logger.DEBUG_FLAG_INFO, "Masking element - generating NO VALUE element") elif (icon_type == Hvf_Plot_Array.PLOT_VALUE): if (Hvf_Plot_Array.PLOT_ELEMENT_BOOLEAN_MASK[y][x]): # This element needs to be detected # Because this step relies on many things going right, possible that our # slices are not amenable to template matching and cause an error # So, try it under a try-except clause. If failure, we place a failure # placeholder to fix later try: cell_object = Hvf_Value.get_value_from_image( cell_slice, cell_slice_backup, plot_type) Logger.get_logger().log_msg( Logger.DEBUG_FLAG_INFO, "Value detected: " + cell_object.get_display_string()) except Exception as e: Logger.get_logger().log_msg( Logger.DEBUG_FLAG_WARNING, "Cell " + str(x) + "," + str(y) + ": Value detection failure") cell_object = Hvf_Value.get_value_from_display_string( Hvf_Value.VALUE_FAILURE) raise Exception(str(e)) else: # This is a no-detect element, so just instantiate a blank: cell_object = Hvf_Value.get_value_from_display_string( Hvf_Value.VALUE_NO_VALUE) Logger.get_logger().log_msg( Logger.DEBUG_FLAG_INFO, "Masking element - generating NO VALUE element") Logger.get_logger().log_msg(Logger.DEBUG_FLAG_INFO, "=====") # Lastly, store into array: plot_values_array[x, y] = cell_object wait_func = (lambda: cv2.waitKey(0)) Logger.get_logger().log_function(Logger.DEBUG_FLAG_DEBUG, wait_func) destroy_windows_func = (lambda: cv2.destroyAllWindows()) Logger.get_logger().log_function(Logger.DEBUG_FLAG_DEBUG, destroy_windows_func) # Return our array: return plot_values_array
def get_plot_grid_lines(plot_image, plot_type, icon_type): Logger.get_logger().log_msg(Logger.DEBUG_FLAG_INFO, "Finding grid lines") plot_w = np.size(plot_image, 1) plot_h = np.size(plot_image, 0) horizontal_img = plot_image.copy() vertical_img = plot_image.copy() # [Horizontal] # Specify size on horizontal axis horizontal_size = horizontal_img.shape[1] # Create structure element for extracting horizontal lines through morphology operations horizontalStructure = cv2.getStructuringElement( cv2.MORPH_RECT, (horizontal_size, 1)) # Apply morphology operations horizontal_img = cv2.morphologyEx(horizontal_img, cv2.MORPH_OPEN, horizontalStructure, iterations=2) #horizontal_img = cv2.erode(horizontal_img, horizontalStructure) #horizontal_img = cv2.dilate(horizontal_img, horizontalStructure) # Then, take a slice from the middle of the plot, and find contours # We will use this to help find grid lines horizontal_slice = Image_Utils.slice_image(horizontal_img, 0, 1, 0.475, 0.05) horizontal_slice = cv2.copyMakeBorder(horizontal_slice, 0, 0, 1, 1, cv2.BORDER_CONSTANT, 0) # Then, find contours (of the blank spaces) and convert to their respective centroid: horizontal_cnts, hierarchy = cv2.findContours(horizontal_slice, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) centroid_horizontal = list( map((lambda c: Hvf_Plot_Array.get_contour_centroid(c)[1] / plot_h), horizontal_cnts)) # [Vertical] # Specify size on vertical axis vertical_size = vertical_img.shape[1] # Create structure element for extracting vertical lines through morphology operations verticalStructure = cv2.getStructuringElement(cv2.MORPH_RECT, (1, vertical_size)) # Apply morphology operations vertical_img = cv2.morphologyEx(vertical_img, cv2.MORPH_OPEN, verticalStructure, iterations=2) #vertical_img = cv2.erode(vertical_img, verticalStructure) #vertical_img = cv2.dilate(vertical_img, verticalStructure) # Then, take a slice from the middle of the plot, and find contours # We will use this to help find grid lines vertical_slice = Image_Utils.slice_image(vertical_img, 0.475, 0.05, 0, 1) vertical_slice = cv2.copyMakeBorder(vertical_slice, 1, 1, 0, 0, cv2.BORDER_CONSTANT, 0) # Then, find contours (of the blank spaces) and convert to their respective centroid: vertical_cnts, hierarchy = cv2.findContours(vertical_slice, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) centroid_vertical = list( map((lambda c: Hvf_Plot_Array.get_contour_centroid(c)[0] / plot_w), vertical_cnts)) # Now, we need to find the grid lines # We assume grid lines are centered in the middle of plot image (since they # are detected that way). Have prelim grid lines, and move then accordingly # to fit into empty spaces # Columns: col_list = [] # Pre-calculate some values: slice_w = np.size(vertical_slice, 1) slice_h = np.size(vertical_slice, 0) for c in range(Hvf_Plot_Array.NUM_OF_PLOT_COLS + 1): # Get our prelim column value: col_val = 0.5 - (0.097 * (5 - c)) # Precalculate our coordinates to check: y = int(slice_h * 0.5) x = int(col_val * slice_w) if (x >= slice_w): x = slice_w - 1 # If this grid line does not coincide with a plot element area, then its good if (vertical_slice[y, x] == 255): # Grid line falls into blank area - we can record value Logger.get_logger().log_msg( Logger.DEBUG_FLAG_INFO, "Prelim column {} grid line works".format(c)) col_list.append(col_val) else: # It coincides -> convert it to the closest centroid of a blank area Logger.get_logger().log_msg( Logger.DEBUG_FLAG_INFO, "Shifting column grid line {} to nearest centroid".format( c)) closest_centroid = list( sorted(centroid_vertical, key=(lambda x: abs(x - col_val))))[0] col_list.append(closest_centroid) # Rows: row_list = [] # Pre-calculate some values: slice_w = np.size(horizontal_slice, 1) slice_h = np.size(horizontal_slice, 0) for r in range(Hvf_Plot_Array.NUM_OF_PLOT_ROWS + 1): # Get our prelim row value: row_val = 0.5 - (0.095 * (5 - r)) # Precalculate our coordinates to check: y = int(row_val * slice_h) x = int(slice_w * 0.5) if (y >= slice_h): y = slice_h - 1 # If this grid line does not coincide with a plot element area, then its good if (horizontal_slice[y, x] == 255): # Grid line falls into blank area - we can record value Logger.get_logger().log_msg( Logger.DEBUG_FLAG_INFO, "Prelim row {} grid line works".format(r)) row_list.append(row_val) else: # It coincides -> convert it to the closest centroid of a blank area Logger.get_logger().log_msg( Logger.DEBUG_FLAG_INFO, "Shifting row grid line {} to nearest centroid".format(r)) closest_centroid = list( sorted(centroid_horizontal, key=(lambda y: abs(y - row_val))))[0] row_list.append(closest_centroid) # Collect our two lists and return them together: return_dict = {} return_dict['row_list'] = row_list return_dict['col_list'] = col_list return return_dict