def draw_results(img_name, element_positions):
img_location = os.path.join(str(Path(__file__).parent.parent.parent),
'resources', 'input_images')
img_location = os.path.join(img_location, img_name)
img = cv2.imread(img_location, cv2.IMREAD_UNCHANGED) # Read the image.
result = img
if len(img.shape) == 3:
trans_mask = img[:, :, 3] == 0 # Remove any transparency.
img[trans_mask] = [255, 255, 255, 255]
img_gray = cv2.cvtColor(
img, cv2.COLOR_BGR2GRAY) # Convert to BR2GRAY (grayscale mode).
th, img_gray = cv2.threshold(img_gray, 127, 255,
cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
result = img_gray
img_gray = result
color = (255, 0, 0)
thickness = 2
for el in element_positions:
start_point = (el[1][0], el[0][0])
end_point = (el[1][1], el[0][1])
img_gray = cv2.rectangle(img_gray, start_point, end_point, color,
thickness)
cv2.imshow("elements", ResizeWithAspectRatio(img_gray, height=900))
cv2.moveWindow('elements', 200, 200)
cv2.waitKey()
construct_output(indent_level="block",
message="Input image processing done.")
def conv_network_analysis(input_image_name):
"""
Main function for convolutional network analysis.
Calls on the network for value recognizing.
Calls on the network for duration analyzing.
:param input_image_name: Name of the image that is being analyzed.
"""
construct_output(indent_level="block",
message="Analyzing the elements of the image ({}) with a convolutional network."
.format(input_image_name))
gpus = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(gpus[0], True)
# Import the dataset and split it to training and testing.
# UNCOMMENT FOR RETRAINING
(test_arr, test_label), (train_arr, train__label) = prepare_new_data(test_data_percentage=0)
value_processing_conv_net.train_note_values_conv_net(test_arr, test_label, train_arr, train__label)
duration_processing_conv_net.train_note_duration_conv_net(test_arr, test_label, train_arr, train__label)
# Load the trained data from the disk.
value_names = value_processing_conv_net.analyze_using_saved_data(input_image_name)
# Load the trained data from the disk.
durations = duration_processing_conv_net.analyze_using_saved_data(input_image_name)
construct_output(indent_level="block",
message="Done analyzing the elements of the image ({}) with a convolutional network."
.format(input_image_name))
return value_names, durations
def generator_main():
"""
Main function for image processing.
Calls on module for row splitting (first), and then module for individual elements extraction(second).
Results are then used for generating the dataset.
"""
# Get the path to the input images.
input_images_path = os.path.join(str(Path(__file__).parent.parent.parent),
'resources', 'input_images')
output_path = os.path.join(str(Path(__file__).parent.parent),
'positions_detection', 'resources', 'train')
output_path_check = [
f for f in listdir(output_path)
if not isfile(join(input_images_path, f))
]
# Get all the images in said folder.
input_images = [
f for f in listdir(input_images_path)
if isfile(join(input_images_path, f))
]
# Iterate through those images.
for input_image in input_images:
if input_image[:
-4] in output_path_check: # Skip already existing images.
continue
input_image_path = os.path.join(
input_images_path,
input_image) # Construct the path to the individual image.
construct_output(
indent_level="block",
message="Processing the resources image ({}).".format(input_image))
row_positions = split_into_rows(
input_image_path) # Firstly, extract rows.
# Then, extract elements from those rows.
x_coords_by_row_number = extract_elements_by_template_matching(
input_image)
element_positions = list(
) # element_positions = list(tuple(Y_UP, Y_DOWN, X_LEFT, X_RIGHT)
for c in x_coords_by_row_number:
element_positions.append((row_positions[c[0]], c[1]))
# draw_results(img_name=input_image, element_positions=element_positions)
# Use the gathered element positions to generate masks for training.
generate_train_element(input_image_path, element_positions)
def train_note_values_conv_net(test_data_arr, test_data_label, train_data_arr,
train_data_label):
"""
This function trains the convolutional network for recognizing note values based on resources data.
Tutorial for this code found here:
https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/keras/classification.ipynb
The results are saved on a disk so that they can be used without retraining the network.
:param train_data_label: Labels with names and durations for the train data images.
:param train_data_arr: Array containing the train images.
:param test_data_label: Labels with names and durations for the test data images.
:param test_data_arr: Array containing the test images.
"""
gpus = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(gpus[0], True)
os.environ[
'TF_CPP_MIN_LOG_LEVEL'] = '2' # Alleged fix for some tensorflow bugs.
construct_output(
indent_level=0,
message="Convolutional Network 1 (Note value determining).")
# Scale these values to a range of 0 to 1 before feeding them to the convolutional network model
print("Scaling test values to [0-1] range.")
test_data_arr = test_data_arr / 255.0
print("Scaling train values to [0-1] range (this will take a while).")
train_data_arr = train_data_arr / 255.0
# Construct the path for saving the results of training.
saved_model_values_path = os.path.abspath(
os.path.join(str(Path(__file__).parent.parent.parent), 'resources'))
saved_model_values_path = os.path.join(saved_model_values_path,
'saved_models')
saved_model_name = "value_processing_net_saved.ckpt"
saved_model_values_path = os.path.join(saved_model_values_path,
saved_model_name)
values_model_cb = tf.keras.callbacks.ModelCheckpoint(
filepath=saved_model_values_path, save_weights_only=True, verbose=1)
# First network only recognizes the values. No need to feed it unrecognized elements (elements with no value).
value_network_train_data_arr = np.array([
x for i, x in enumerate(train_data_arr)
if train_data_label[i][0][0] != "Uncategorized"
])
value_network_train_data_label = np.array([(x[0][0], x[1])
for x in train_data_label
if x[0][0] != "Uncategorized"])
value_network_test_data_arr = np.array([
x for i, x in enumerate(test_data_arr)
if test_data_label[i][0][0] != "Uncategorized"
])
value_network_test_data_label = np.array([(x[0][0], x[1])
for x in test_data_label
if x[0][0] != "Uncategorized"])
class_names = [
"A3",
"A4",
"A5", # class_names contains possible results
"B3",
"B4",
"B5",
"C3",
"C4",
"C5",
"D3",
"D4",
"D5",
"E3",
"E4",
"E5",
"F3",
"F4",
"F5",
"G3",
"G4",
"G5"
]
# Fetch only the labels (note values) from the data.
value_network_train_data_label = [
item[0] for item in value_network_train_data_label
]
# Assign the corresponding numerical values to labels.
value_network_train_data_label_values_numerical = values_to_numerical(
value_network_train_data_label, class_names)
with tf.device(
'/GPU:1'
): # Specify using nvidia discrete GPU instead of Intel integrated graphics.
construct_output(indent_level=0, message="Start training.")
# Set up the layers.
# The first layer in this network, tf.keras.layers.Flatten, transforms the format of the images
# from a 2D array(200x200px) to 1D array(of 200x200 = 40000 pixels)
# After the pixels are flattened, the network consists of a sequence of two tf.keras.layers.Dense layers.
# These are densely connected, or fully connected, neural layers.
# The first Dense layer has 128 nodes( or neurons).
# The second( and last) layer returns an array with length of 22.
# Each node contains a score that indicates the current image belongs to one of the 22 classes.
model = tf.keras.Sequential([
tf.keras.layers.Flatten(input_shape=(200, 200)),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dense(22)
])
# Before the model is ready for training, it needs a few more settings.
# These are added during the model's compile step:
# Loss function —This measures how accurate the model is during training.
# You want to minimize this function to "steer" the model in the right direction.
# Optimizer —This is how the model is updated based on the data it sees and its loss function.
# Metrics —Used to monitor the training and testing steps.
# The following example uses accuracy, the fraction of the images that are correctly classified.
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
metrics=['accuracy'])
# Training the convolutional network model requires the following steps:
# Feed the training data to the model.
# In this example, the training data is in the train_images and train_labels arrays.
# The model learns to associate images and labels.
# You ask the model to make predictions about a test set—in this example, the test_images array.
# Verify that the predictions match the labels from the test_labels array.
model.fit(value_network_train_data_arr,
value_network_train_data_label_values_numerical,
epochs=3,
callbacks=[values_model_cb])
construct_output(
indent_level=0,
message="Save the network weights to avoid retraining on every run."
)
# Attach a softmax layer to convert the logits to probabilities, which are easier to interpret.
probability_model = tf.keras.Sequential(
[model, tf.keras.layers.Softmax()])
# TESTING THE NETWORK. =======================================================================================
# Compare how the model performs on the test dataset.
# value_network_test_data_label = [item[0] for item in value_network_test_data_label]
# value_network_test_data_label_values_numerical = values_to_numerical(
# value_network_test_data_label,
# class_names)
# test_loss, test_acc = model.evaluate(value_network_test_data_arr,
# value_network_test_data_label_values_numerical,
# verbose=2
# )
# print('\nTest accuracy:', test_acc)
# predictions = probability_model.predict(value_network_test_data_arr)
# print(predictions[0])
# print("max= ", np.argmax(predictions[0]))
# import cv2
# cv2.imshow("img", value_network_test_data_arr[0])
# cv2.waitKey()
construct_output(indent_level=0, message="End training.")
construct_output(
indent_level=0,
message="Convolutional Network 1 (Note value determining) Done.")
def extract_elements_by_template_matching(img_name):
"""
Main function for element extraction that calls on all the sub-functions.
:param img_name: Name of the resources image from which image rows where extracted.
"""
img_location = os.path.join(str(Path(__file__).parent.parent.parent),
'resources', 'input_images')
img_location = os.path.join(img_location, 'input_images_rows',
img_name[:-4])
rows_numerated = [
row for row in os.listdir(img_location) if row.startswith("row")
]
construct_output(indent_level=0,
message="Finding individual elements in the saved rows.")
construct_output(indent_level=1, message="Reading extracted rows.")
# element_positions = list(tuple(ROW_NUMBER, X_LEFT, X_RIGHT)
element_positions = []
for row_number, row_img in enumerate(rows_numerated):
construct_output(indent_level=2,
message="Reading row number {}.".format(row_number))
img_rgb = cv2.imread(img_location + "/" + row_img) # Read the image.
img_gray = cv2.cvtColor(
img_rgb, cv2.COLOR_BGR2GRAY) # Convert it into a gray image.
image_w, image_h = img_gray.shape[::-1] # Image dimensions.
# Construct the path to the templates.
template_path = os.path.abspath(
os.path.join(str(Path(__file__).parent.parent.parent), 'resources',
'templates'))
# List all the templates that are not within the 'line_start_templates' subdirectory.
template_names = [
t for t in os.listdir(template_path)
if not str(t).startswith("line_start_templates")
]
# (1) Find templates using template matching.
# Use the 'match_templates' function to get the locations(list),dimensions(list).
# Also, get list of booleans on whether the templates are recognized by their names (such as clef_g),
# or if they need to be processed by a conv. network.
# Replace the values in 'template_names' with names of the found templates
t_loc, t_dim, t_recognized_list, found_t_names = match_templates(
template_names, template_path, img_gray)
# (2) Get the start and end coordinates of the templates.
construct_output(
indent_level=2,
message="Matching the row elements with the templates.")
templates_start_end = [(x[0], x[0] + t_dim[index][0])
for index, x in enumerate(t_loc)]
# (3) Save the images in the standard size (200x200 px). Return value only used for visualisation.
construct_output(
indent_level=2,
message="Saving found elements in the row {}.".format(row_number))
x_coords = find_x_coords(templates_start_end)
for x_coord in x_coords:
element_positions.append((row_number, x_coord))
construct_output(
indent_level=0,
message="Finding individual elements in the saved rows done.")
# RESULT DRAWING (TESTING).
# from copy import deepcopy
# from note_recognition_app.image_segmentation_dataset_generator.img_resizer import ResizeWithAspectRatio
# from note_recognition_app.image_segmentation_dataset_generator.row_splitter_result_visualizer import generate_result_img
# Draw the results of (2). Leave for checking purposes.
# tmp_img = deepcopy(img_rgb)
# for el in templates_start_end:
# cv2.rectangle(tmp_img, (el[0], 20), (el[1], image_h - 20), (255, 0, 255), 2)
# cv2.imshow('Found elements.', ResizeWithAspectRatio(tmp_img, width=1000))
# cv2.moveWindow('Found elements.', 0, 200)
# Draw the results of (3). Leave for checking purposes.
# result_img = generate_result_img(input_images_individual_elements)
# cv2.imshow('Final Result.', ResizeWithAspectRatio(result_img, width=500))
# cv2.moveWindow('Final Result.', 0, 400)
# cv2.waitKey()
return element_positions
def split_into_rows(img_path):
"""
This function splits the resources image into separate rows of note lines.
:param img_path: Path to the image.
:return: boolean: True if successful, false otherwise.
"""
try:
construct_output(indent_level=0, message="Row splitting.")
img_name = img_path[img_path.rfind('\\') + 1:] # Extract image name from the given path.
# Directory name for the directory that will hold the rows of the resources image.
dir_name = os.path.join(str(Path(__file__).parent.parent.parent), 'resources', 'input_images')
dir_name = os.path.join(dir_name, 'input_images_rows', img_name[:-4])
try: # Try creating a directory.
construct_output(indent_level=1, message="Creating a folder for the image: {}".format(dir_name))
os.mkdir(dir_name)
except OSError as e:
construct_output(indent_level=1, message="Folder already exists: {}".format(dir_name))
construct_output(indent_level=1, message="Reading the resources image {}.".format(img_name))
img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED) # Read the image.
trans_mask = img[:, :, 3] == 0 # Remove any transparency.
img[trans_mask] = [255, 255, 255, 255]
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # Convert to BR2GRAY (grayscale mode).
# Make a black and white image based on a threshold.
th, img_gray = cv2.threshold(img_gray, 127, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
image_w, image_h = img_gray.shape[::-1] # Image dimensions.
template_path = os.path.abspath(
os.path.join(str(Path(__file__).parent.parent.parent), 'resources', 'templates', 'line_start_templates'))
row_templates = [file for file in os.listdir(template_path)]
construct_output(indent_level=1, message="Finding rows in the image.")
t_locations = [] # List that will contain all the locations that were found by template matching.
t_dimensions = [] # List that will contain all the dimensions of the found templates on the locations.
for t in row_templates: # Iterate through all of the vertical line templates.
template = cv2.imread(template_path + "\\" + t, 0) # Read the template from the path.
res = cv2.matchTemplate(img_gray, template, cv2.TM_CCOEFF_NORMED) # Convert the template into a gray image.
threshold = 0.80 # The threshold to determine whether a part of an image is similar enough to the template.
locations = np.where(res >= threshold) # Locations in the image where the template matching found results.
template_w, template_h = template.shape[::-1] # Dimensions of the current template.
# list(zip(*locations[::-1])) -> Match the 'x' and 'y' coordinates into a tuple them and save into a list.
# Iterate through locations to remove elements already found by previous templates.
for point in list(zip(*locations[::-1])):
if len(t_locations) == 0: # Save the first template matching results without checking.
t_locations.append(point)
t_dimensions.append((template_w, template_h)) # Also save the template dimensions.
else: # Check if 'v_line_locations' already contains the new point +/- 6 px, don't add if true.
if not np.intersect1d(list(zip(*t_locations))[1], list(range(point[1] - 6, point[1] + 6))):
t_locations.append(point)
t_dimensions.append((template_w, template_h))
construct_output(indent_level=1, message="Saving the found rows into folder: {}".format(dir_name))
row_positions = list()
for index, el in enumerate(t_locations): # Iterate through found locations.
row_position = tuple((el[1] - 40, el[1] + t_dimensions[index][1] + 40))
row_positions.append(row_position)
img_slice_name_and_path = dir_name + "/row" + str(index) + ".png" # Generate a path and a name.
img_slice = img_gray[el[1] - 40:el[1] + t_dimensions[index][1] + 40:, 0:image_w] # Cut the part of the img.
cv2.imwrite(img_slice_name_and_path, img_slice) # Save that part of the image.
return row_positions
except Exception as e: # Catch exception.
print(e)
exit(-1) # If any exceptions caught, return False.
construct_output(indent_level=0, message="Row splitting done.")
def generate_train_element(input_image_path, element_positions):
"""
For input image and element positions on the input image, generate masks using those positions.
# Standard image size is 2048 x 2048 px when saving.
"""
STANDARD_IMG_DIM = 4096
construct_output(
indent_level=0,
message="Row Generating the dataset for finding element positions.")
# Get the image name from the input path.
img_name = input_image_path.split('\\')[-1][:-4]
construct_output(indent_level=1,
message="Adding {} to the dataset".format(img_name))
# Generate the path to the training directory.
train_dataset_input = os.path.join(str(Path(__file__).parent.parent),
'positions_detection', 'resources',
'train', img_name)
# Generate the path to the original image directory.
images_folder = os.path.join(train_dataset_input, 'images')
# Generate the path to the masks directory.
masks_folder = os.path.join(train_dataset_input, 'masks')
construct_output(indent_level=2,
message="Creating the needed directories.")
try: # Generate needed directory structure.
os.mkdir(train_dataset_input)
os.mkdir(images_folder)
os.mkdir(masks_folder)
except FileExistsError as _:
pass
construct_output(indent_level=2,
message="Saving a non-transparent image into {}.".format(
images_folder))
# Get the original image.
org_img = cv2.imread(input_image_path, cv2.IMREAD_UNCHANGED)
org_img = ResizeWithAspectRatio(
org_img, width=2600) # Resize to standard dimensions.
img_height, img_width = org_img.shape[:
-1] # Get the dimensions of the original image.
img_gray = deepcopy(org_img)
if len(org_img.shape) == 3: # If the image has transparency.
trans_mask = org_img[:, :, 3] == 0 # Remove any transparency.
org_img[trans_mask] = [255, 255, 255, 255]
img_gray = cv2.cvtColor(
org_img,
cv2.COLOR_BGR2GRAY) # Convert to BR2GRAY (grayscale mode).
th, img_gray = cv2.threshold(img_gray, 127, 255,
cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
if img_height < STANDARD_IMG_DIM: # Resize height to standard dimensions by adding padding on the bottom.
additional_lines_h = STANDARD_IMG_DIM - img_height # Calculate the needed padding.
additional_lines = np.zeros((additional_lines_h, img_width),
np.uint8) # Make the padding black.
img_gray = np.concatenate((img_gray, additional_lines),
axis=0) # Add the padding.
if img_width < STANDARD_IMG_DIM:
additional_lines_w = STANDARD_IMG_DIM - img_width # Calculate the needed padding.
additional_lines = np.zeros((STANDARD_IMG_DIM, additional_lines_w),
np.uint8) # Make the padding black.
img_gray = np.concatenate((img_gray, additional_lines),
axis=1) # Add the padding.
img_height, img_width = img_gray.shape # Get the NEW dimensions of the original image.
img_name = img_name + '.png'
img_name = os.path.join(images_folder, img_name)
# Save the non-transparent original image into the dataset.
cv2.imwrite(img_name, ResizeWithAspectRatio(img_gray, width=2048))
# Construct a black image with the same dimensions.
black_image = np.zeros((img_height, img_width), np.uint8)
# Iterate over found elements.
construct_output(indent_level=2, message="***Creating masks.***")
for index, element_position in enumerate(element_positions):
construct_output(indent_level=2,
message="Creating mask {}.".format(index))
# Generate a a copy of a black image.
temp_black_img = deepcopy(black_image)
# Get the individual element position.
y_start = element_position[0][0]
y_end = element_position[0][1]
x_start = element_position[1][0]
x_end = element_position[1][1]
# Extract the element.
element = img_gray[y_start:y_end, x_start:x_end]
# Make a white rectangle over the element position.
element.fill(255)
# Save that rectangle onto the black image.
temp_black_img[y_start:y_end, x_start:x_end] = img_gray[y_start:y_end,
x_start:x_end]
# Generate mask name
# (mask being the black image containing a single white rectangle in the place of the element).
mask_name = 'mask' + str(index) + '.png'
mask_name = os.path.join(masks_folder, mask_name)
# Save the mask.
cv2.imwrite(mask_name, ResizeWithAspectRatio(temp_black_img,
width=2048))
construct_output(indent_level=3,
message="Mask {} saved into {}.".format(
'mask' + str(index) + '.png', masks_folder))