def get_plot(hvf_image_gray, y_ratio, y_size, x_ratio, x_size, plot_type,
icon_type):
plot_image = Image_Utils.slice_image(hvf_image_gray, y_ratio, y_size,
x_ratio, x_size)
hvf_image_gray_process = Image_Utils.preprocess_image(
hvf_image_gray.copy())
plot_image_process = Image_Utils.slice_image(hvf_image_gray_process,
y_ratio, y_size, x_ratio,
x_size)
# Get bounding box from processed image:
top_left, w, h = Hvf_Plot_Array.get_bounding_box(plot_image_process)
bottom_right = (top_left[0] + w, top_left[1] + h)
# Need to specifically handle raw value plot - can have a discontinuity in the
# x axis (with triangle icon), which causes a mis-fit. So need to fill in x-axis
# and try again
cv2.line(plot_image_process, (top_left[0], top_left[1] + int(h / 2)),
(top_left[0] + w, top_left[1] + int(h / 2)), (0),
max(int(h * 0.015), 1))
top_left, w, h = Hvf_Plot_Array.get_bounding_box(plot_image_process)
bottom_right = (top_left[0] + w, top_left[1] + h)
# For debugging: Draw rectangle around the plot - MUST BE COMMENTED OUT, BECAUSE
# IT WILL INTERFERE WITH LATER PLOT EXTRACTIONS
#cv2.rectangle(plot_image, top_left, bottom_right, 0, 2)
# Debug function for showing the plot:
#show_plot_func = (lambda : cv2.imshow("Bound rect for plot " + plot_type, plot_image))
#Logger.get_logger().log_function(Logger.DEBUG_FLAG_DEBUG, show_plot_func);
#cv2.waitKey();
# Slice out the axes plot on the original:
tight_plot = plot_image[top_left[1]:(top_left[1] + h),
top_left[0]:(top_left[0] + w)]
# And extract the values from the array:
plot_array = Hvf_Plot_Array.extract_values_from_plot(
tight_plot, plot_type, icon_type)
# Return the array:
return plot_array, tight_plot
def is_pattern_not_shown(hvf_image_gray, y_ratio, y_size, x_ratio, x_size):
# Calculate height/width for calculation later:
height = np.size(hvf_image_gray, 0)
width = np.size(hvf_image_gray, 1)
# Slice image:
hvf_image_gray = Image_Utils.preprocess_image(hvf_image_gray)
sliced_img = Image_Utils.slice_image(hvf_image_gray, y_ratio, y_size,
x_ratio, x_size)
# Try to detect a bounding box:
top_left, w, h = Hvf_Plot_Array.get_bounding_box(sliced_img)
# Calculate the relative (percentage) size of the bounding box compared to slice:
box_ratio_w = w / (x_size * width)
box_ratio_h = h / (y_size * height)
# Define a threshold below which if the size ratio is, we declare that the pattern
# is not detected:
threshold_size = 0.3
return (box_ratio_w < threshold_size or box_ratio_h < threshold_size)
def identify_digit(plot_element, allow_search_zero):
# We template match against all icons and look for best fit:
best_match = None
best_val = None
best_loc = None
best_scale_factor = None
height = np.size(plot_element, 0)
width = np.size(plot_element, 1)
# Can skip 0 if flag tells us to. This can help maximize accuracy in low-res cases
# Do this when we know something about the digit (it is a leading digit, etc)
start_index = 0
if not allow_search_zero:
start_index = 1
for ii in range(start_index,
len(Hvf_Value.value_icon_templates.keys())):
for dir in Hvf_Value.value_icon_templates[ii]:
# First, scale our template value:
val_icon = Hvf_Value.value_icon_templates[ii][dir]
plot_element_temp = plot_element.copy()
scale_factor = 1
# Use the smaller factor to make sure we fit into the element icon
if (height < np.size(val_icon, 0)):
# Need to upscale plot_element
scale_factor = np.size(val_icon, 0) / height
plot_element_temp = cv2.resize(plot_element_temp, (0, 0),
fx=scale_factor,
fy=scale_factor)
else:
# Need to upscale val_icon
scale_factor = height / (np.size(val_icon, 0))
val_icon = cv2.resize(val_icon, (0, 0),
fx=scale_factor,
fy=scale_factor)
# In case the original is too small by width compared to value_icon, need
# to widen - do so by copymakeborder replicate
if (np.size(plot_element_temp, 1) < np.size(val_icon, 1)):
border = np.size(val_icon, 1) - np.size(
plot_element_temp, 1)
#plot_element_temp = cv2.copyMakeBorder(plot_element_temp,0,0,0,border,cv2.BORDER_CONSTANT,0);
# Apply template matching:
temp_matching = cv2.matchTemplate(plot_element_temp, val_icon,
cv2.TM_CCOEFF_NORMED)
# Grab our result
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(
temp_matching)
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_DEBUG,
"Matching against " + str(ii) + ": " + str(max_val))
# Check to see if this is our best fit yet:
if (best_match is None or max_val > best_match):
# This is best fit - record the match value and the actual value
best_match = max_val
best_val = ii
best_loc = max_loc
best_scale_factor = scale_factor
# TODO: refine specific cases that tend to be misclassified
# 1 vs 4
if (best_val == 4 or best_val == 1):
# Cut number in half and take bottom half -> find contours
# If width of contour is most of element --> 4
# otherwise, 1
bottom_half = Image_Utils.slice_image(plot_element, 0.5, 0.5, 0, 1)
cnts, hierarchy = cv2.findContours(
cv2.bitwise_not(bottom_half.copy()), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
# Sort contours by width
largest_contour = sorted(cnts,
key=Hvf_Value.contour_width,
reverse=True)[0]
if (Hvf_Value.contour_width(largest_contour) > width * 0.8):
best_val = 4
else:
best_val = 1
return best_val, best_loc, best_scale_factor, best_match
def extract_values_from_plot(plot_image, plot_type, icon_type):
# First, image process for best readability:
#plot_image = cv2.GaussianBlur(plot_image, (5,5), 0)
plot_image_backup = plot_image.copy()
# Perform image processing depending on plot type:
if (icon_type == Hvf_Plot_Array.PLOT_PERC):
plot_image = cv2.bitwise_not(
cv2.adaptiveThreshold(plot_image, 255,
cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
cv2.THRESH_BINARY_INV, 11, 5))
elif (icon_type == Hvf_Plot_Array.PLOT_VALUE):
#plot_image = cv2.GaussianBlur(plot_image, (5,5), 0)
ret2, plot_image = cv2.threshold(
plot_image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
kernel_size = 31
mean_offset = 15
plot_image_backup = cv2.bitwise_not(
cv2.adaptiveThreshold(plot_image_backup, 255,
cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
cv2.THRESH_BINARY_INV, kernel_size,
mean_offset))
kernel = np.ones((3, 3), np.uint8)
# For readability, grab our height/width:
plot_width = np.size(plot_image, 1)
plot_height = np.size(plot_image, 0)
# The elements are laid out roughly within a 10x10 grid
NUM_CELLS_ROW = Hvf_Plot_Array.NUM_OF_PLOT_ROWS
NUM_CELLS_COL = Hvf_Plot_Array.NUM_OF_PLOT_COLS
# Delete triangle icon, if we can find it:
if (plot_type == Hvf_Plot_Array.PLOT_RAW):
if not (Hvf_Plot_Array.find_and_delete_triangle_icon(
plot_image, "v1")):
Hvf_Plot_Array.find_and_delete_triangle_icon(plot_image, "v2")
# Mask out corners:
corner_mask = Hvf_Plot_Array.generate_corner_mask(
plot_width, plot_height)
plot_image = cv2.bitwise_or(plot_image, cv2.bitwise_not(corner_mask))
# First, declare our return value array, no need to really initialize bc we'll
# iterate through it
plot_values_array = 0
if (icon_type == Hvf_Plot_Array.PLOT_PERC):
plot_values_array = np.zeros((NUM_CELLS_COL, NUM_CELLS_ROW),
dtype=Hvf_Perc_Icon)
elif (icon_type == Hvf_Plot_Array.PLOT_VALUE):
plot_values_array = np.zeros((NUM_CELLS_COL, NUM_CELLS_ROW),
dtype=Hvf_Value)
plot_image = Hvf_Plot_Array.delete_plot_axes(plot_image)
# Grab the grid lines:
grid_line_dict = Hvf_Plot_Array.get_plot_grid_lines(
plot_image, plot_type, icon_type)
plot_image_debug_copy = plot_image.copy()
# Debug code - draws out slicing for the elements on the plot:
for c in range(Hvf_Plot_Array.NUM_OF_PLOT_COLS + 1):
x = int(grid_line_dict['col_list'][c] * plot_width)
#cv2.line(plot_image_debug_copy, (x, 0), (x, plot_height), (0), 1);
for r in range(Hvf_Plot_Array.NUM_OF_PLOT_ROWS + 1):
y = int(grid_line_dict['row_list'][r] * plot_height)
#cv2.line(plot_image_debug_copy, (0, y), (plot_width, y), (0), 1);
# Debug function for showing the plot:
show_plot_func = (
lambda: cv2.imshow("plot " + icon_type, plot_image_debug_copy))
Logger.get_logger().log_function(Logger.DEBUG_FLAG_DEBUG,
show_plot_func)
#cv2.imshow("plot " + icon_type, plot_image_debug_copy)
#cv2.waitKey();
# We iterate through our array, then slice out the appropriate cell from the plot
for x in range(0, NUM_CELLS_COL):
for y in range(0, NUM_CELLS_ROW):
# Debug info for indicating what cell we're computing:
Logger.get_logger().log_msg(Logger.DEBUG_FLAG_INFO,
"Cell " + str(x) + "," + str(y))
# Grab our cell slice for the plot element
# (remember arguments: slice_image(image, y_ratio, y_size, x_ratio, x_size):
# The height of the axes tends to extend ~2.5% past the elements on top, bottom
# The width of the axes tends to extend
# So we take that into account when we take the slice
row_grid_val = grid_line_dict['row_list'][y]
row_grid_val_size = grid_line_dict['row_list'][
y + 1] - grid_line_dict['row_list'][y]
col_grid_val = grid_line_dict['col_list'][x]
col_grid_val_size = grid_line_dict['col_list'][
x + 1] - grid_line_dict['col_list'][x]
cell_slice = Image_Utils.slice_image(plot_image, row_grid_val,
row_grid_val_size,
col_grid_val,
col_grid_val_size)
cell_slice_backup = Image_Utils.slice_image(
plot_image_backup, row_grid_val, row_grid_val_size,
col_grid_val, col_grid_val_size)
cell_object = 0
# Then, need to analyze to figure out what element is in this position
# What we look for depends on type of plot - perc vs value
if (icon_type == Hvf_Plot_Array.PLOT_PERC):
if (Hvf_Plot_Array.PLOT_ELEMENT_BOOLEAN_MASK[y][x]):
# This element needs to be detected
# Because this step relies on many things going right, possible that our
# slices are not amenable to template matching and cause an error
# So, try it under a try-except clause. If failure, we place a failure
# placeholder
try:
cell_object = Hvf_Perc_Icon.get_perc_icon_from_image(
cell_slice)
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_INFO,
"Percentile Icon detected: " +
cell_object.get_display_string())
except:
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_WARNING,
"Cell " + str(x) + "," + str(y) +
": Percentile icon detection failure")
cell_object = Hvf_Perc_Icon.get_perc_icon_from_char(
Hvf_Perc_Icon.PERC_FAILURE_CHAR)
raise Exception(str(e))
else:
# This is a no-detect element, so just instantiate a blank:
cell_object = Hvf_Perc_Icon.get_perc_icon_from_char(
Hvf_Perc_Icon.PERC_NO_VALUE_CHAR)
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_INFO,
"Masking element - generating NO VALUE element")
elif (icon_type == Hvf_Plot_Array.PLOT_VALUE):
if (Hvf_Plot_Array.PLOT_ELEMENT_BOOLEAN_MASK[y][x]):
# This element needs to be detected
# Because this step relies on many things going right, possible that our
# slices are not amenable to template matching and cause an error
# So, try it under a try-except clause. If failure, we place a failure
# placeholder to fix later
try:
cell_object = Hvf_Value.get_value_from_image(
cell_slice, cell_slice_backup, plot_type)
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_INFO, "Value detected: " +
cell_object.get_display_string())
except Exception as e:
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_WARNING, "Cell " + str(x) +
"," + str(y) + ": Value detection failure")
cell_object = Hvf_Value.get_value_from_display_string(
Hvf_Value.VALUE_FAILURE)
raise Exception(str(e))
else:
# This is a no-detect element, so just instantiate a blank:
cell_object = Hvf_Value.get_value_from_display_string(
Hvf_Value.VALUE_NO_VALUE)
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_INFO,
"Masking element - generating NO VALUE element")
Logger.get_logger().log_msg(Logger.DEBUG_FLAG_INFO, "=====")
# Lastly, store into array:
plot_values_array[x, y] = cell_object
wait_func = (lambda: cv2.waitKey(0))
Logger.get_logger().log_function(Logger.DEBUG_FLAG_DEBUG, wait_func)
destroy_windows_func = (lambda: cv2.destroyAllWindows())
Logger.get_logger().log_function(Logger.DEBUG_FLAG_DEBUG,
destroy_windows_func)
# Return our array:
return plot_values_array
def get_plot_grid_lines(plot_image, plot_type, icon_type):
Logger.get_logger().log_msg(Logger.DEBUG_FLAG_INFO,
"Finding grid lines")
plot_w = np.size(plot_image, 1)
plot_h = np.size(plot_image, 0)
horizontal_img = plot_image.copy()
vertical_img = plot_image.copy()
# [Horizontal]
# Specify size on horizontal axis
horizontal_size = horizontal_img.shape[1]
# Create structure element for extracting horizontal lines through morphology operations
horizontalStructure = cv2.getStructuringElement(
cv2.MORPH_RECT, (horizontal_size, 1))
# Apply morphology operations
horizontal_img = cv2.morphologyEx(horizontal_img,
cv2.MORPH_OPEN,
horizontalStructure,
iterations=2)
#horizontal_img = cv2.erode(horizontal_img, horizontalStructure)
#horizontal_img = cv2.dilate(horizontal_img, horizontalStructure)
# Then, take a slice from the middle of the plot, and find contours
# We will use this to help find grid lines
horizontal_slice = Image_Utils.slice_image(horizontal_img, 0, 1, 0.475,
0.05)
horizontal_slice = cv2.copyMakeBorder(horizontal_slice, 0, 0, 1, 1,
cv2.BORDER_CONSTANT, 0)
# Then, find contours (of the blank spaces) and convert to their respective centroid:
horizontal_cnts, hierarchy = cv2.findContours(horizontal_slice,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
centroid_horizontal = list(
map((lambda c: Hvf_Plot_Array.get_contour_centroid(c)[1] / plot_h),
horizontal_cnts))
# [Vertical]
# Specify size on vertical axis
vertical_size = vertical_img.shape[1]
# Create structure element for extracting vertical lines through morphology operations
verticalStructure = cv2.getStructuringElement(cv2.MORPH_RECT,
(1, vertical_size))
# Apply morphology operations
vertical_img = cv2.morphologyEx(vertical_img,
cv2.MORPH_OPEN,
verticalStructure,
iterations=2)
#vertical_img = cv2.erode(vertical_img, verticalStructure)
#vertical_img = cv2.dilate(vertical_img, verticalStructure)
# Then, take a slice from the middle of the plot, and find contours
# We will use this to help find grid lines
vertical_slice = Image_Utils.slice_image(vertical_img, 0.475, 0.05, 0,
1)
vertical_slice = cv2.copyMakeBorder(vertical_slice, 1, 1, 0, 0,
cv2.BORDER_CONSTANT, 0)
# Then, find contours (of the blank spaces) and convert to their respective centroid:
vertical_cnts, hierarchy = cv2.findContours(vertical_slice,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
centroid_vertical = list(
map((lambda c: Hvf_Plot_Array.get_contour_centroid(c)[0] / plot_w),
vertical_cnts))
# Now, we need to find the grid lines
# We assume grid lines are centered in the middle of plot image (since they
# are detected that way). Have prelim grid lines, and move then accordingly
# to fit into empty spaces
# Columns:
col_list = []
# Pre-calculate some values:
slice_w = np.size(vertical_slice, 1)
slice_h = np.size(vertical_slice, 0)
for c in range(Hvf_Plot_Array.NUM_OF_PLOT_COLS + 1):
# Get our prelim column value:
col_val = 0.5 - (0.097 * (5 - c))
# Precalculate our coordinates to check:
y = int(slice_h * 0.5)
x = int(col_val * slice_w)
if (x >= slice_w):
x = slice_w - 1
# If this grid line does not coincide with a plot element area, then its good
if (vertical_slice[y, x] == 255):
# Grid line falls into blank area - we can record value
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_INFO,
"Prelim column {} grid line works".format(c))
col_list.append(col_val)
else:
# It coincides -> convert it to the closest centroid of a blank area
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_INFO,
"Shifting column grid line {} to nearest centroid".format(
c))
closest_centroid = list(
sorted(centroid_vertical,
key=(lambda x: abs(x - col_val))))[0]
col_list.append(closest_centroid)
# Rows:
row_list = []
# Pre-calculate some values:
slice_w = np.size(horizontal_slice, 1)
slice_h = np.size(horizontal_slice, 0)
for r in range(Hvf_Plot_Array.NUM_OF_PLOT_ROWS + 1):
# Get our prelim row value:
row_val = 0.5 - (0.095 * (5 - r))
# Precalculate our coordinates to check:
y = int(row_val * slice_h)
x = int(slice_w * 0.5)
if (y >= slice_h):
y = slice_h - 1
# If this grid line does not coincide with a plot element area, then its good
if (horizontal_slice[y, x] == 255):
# Grid line falls into blank area - we can record value
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_INFO,
"Prelim row {} grid line works".format(r))
row_list.append(row_val)
else:
# It coincides -> convert it to the closest centroid of a blank area
Logger.get_logger().log_msg(
Logger.DEBUG_FLAG_INFO,
"Shifting row grid line {} to nearest centroid".format(r))
closest_centroid = list(
sorted(centroid_horizontal,
key=(lambda y: abs(y - row_val))))[0]
row_list.append(closest_centroid)
# Collect our two lists and return them together:
return_dict = {}
return_dict['row_list'] = row_list
return_dict['col_list'] = col_list
return return_dict