def image_processing(image):
PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
image_path = os.path.dirname(PROJECT_ROOT)+"\media"+"\\"+str(image)
image = cv2.imread(image_path)
angle, rotated = correct_skew(image)
gray = cv2.cvtColor(rotated,cv2.COLOR_BGR2GRAY)
thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (40,1))
remove_horizontal = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, horizontal_kernel, iterations=2)
cnts = cv2.findContours(remove_horizontal, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
for c in cnts:
cv2.drawContours(rotated, [c], -1, (255,255,255), 5)
vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1,40))
remove_vertical = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, vertical_kernel, iterations=2)
cnts = cv2.findContours(remove_vertical, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
for c in cnts:
cv2.drawContours(rotated, [c], -1, (255,255,255), 5)
gray = cv2.cvtColor(rotated,cv2.COLOR_BGR2GRAY)
filter1 = cv2.medianBlur(gray,5)
filter2 = cv2.GaussianBlur(filter1,(5,5),0)
dst = cv2.fastNlMeansDenoising(filter2,None,17,9,17)
th1 = cv2.adaptiveThreshold(dst,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,11,2)
return th1
def scan_boxscore(img):
# print(img)
edged = cv2.Canny(img, 10, 250)
dst = cv2.fastNlMeansDenoising(img, None, 10, 10, 100)
(cnts, _) = cv2.findContours(edged.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
idx = 0
for c in cnts:
x,y,w,h = cv2.boundingRect(c)
if w>150 and h>120 :
if w<165 and h<135 :
new_img2=img[y:y+h,x:x+w]
return new_img2
def processImage(image, thresholdValue):
img = cv2.imread(image)
grayScale = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
denoised = cv2.fastNlMeansDenoising(grayScale, h=5)
result = 255 - denoised
processedImage = cv2.threshold(result, thresholdValue, 255,
cv2.THRESH_BINARY)[1]
processedImageClone = cv2.cvtColor(processedImage, cv2.COLOR_GRAY2BGR)
thresh = cv2.threshold(processedImage, thresholdValue, 255, 0)[1]
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, 2)
for i in range(0, shape(contours)[0]):
x, y, w, h = cv2.boundingRect(contours[i])
img = cv2.rectangle(processedImageClone, (x, y), (x + w, y + h),
(0, 255, 0), 2)
return processedImageClone
def scan_boxnumber(image):
edged = cv2.Canny(image, 10, 250)
dst = cv2.fastNlMeansDenoising(image, None, 10, 10, 100)
(cnts, _) = cv2.findContours(edged.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
idx = 0
for c in cnts:
x,y,w,h = cv2.boundingRect(c)
if w>550 and h>60 :
if w<610 and h<90 :
idx+=1
new_img=image[y:y+h,x:x+w]
return new_img
def main(image):
images = list()
# Read the image using Opencv library.
img = cv.imread(image)
# Conver the image in black and night with a grayscale.
imgToGray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
images.append(imgToGray)
# Denoise the image from potentiel noise.
imgDst = cv.fastNlMeansDenoising(imgToGray)
# Blur the image using a Gaussian filter of 5 x 5.
imgBlured = cv.GaussianBlur(imgDst, (5, 5), 0)
images.append(imgBlured)
# Filter the image to get the edges.
imgSobel, angle = sobelFilter(imgBlured)
# Thin the edges with a nms filter.
imgNMS = nonMaximumSupression(imgSobel, angle)
images.append(imgNMS)
# Use a hysteris threshold to transform weak pixel into strong one if needed.
# tmp, weak, strong = threshold(imgNMS)
# final = hysteresis(tmp, weak, strong)
final = hysteresisThresholding(imgNMS)
images.append(final)
# Real Canny from Opencv library. Added in the final images to compare result.
# Use auto Canny to detect best weak and strong threshold, based on this blog:
# https://www.pyimagesearch.com/2015/04/06/zero-parameter-automatic-canny-edge-detection-with-python-and-opencv/
median = np.median(imgToGray)
lower = int(max(0, (1.0 - 0.33) * median))
upper = int(min(255, (1.0 + 0.33) * median))
imgCanny = cv.Canny(imgToGray, lower, upper)
images.append(imgCanny)
# Display the image transition.
show_images(images)
def DetectColorGrids(Color_checker_image):
img_gray = cv.cvtColor(Color_checker_image,cv.COLOR_BGR2GRAY)
img_denoise = cv.fastNlMeansDenoising(img_gray,10,7,21)
img_threshold = cv.adaptiveThreshold(img_denoise,255,cv.ADAPTIVE_THRESH_MEAN_C,cv.THRESH_BINARY,21,3)
kernel = np.ones((3,3), np.uint8)
img_eroded = cv.erode(img_threshold,kernel)
small_grid_size_range = GetSmallGridSizeRange(Color_checker_image)
contours, _hierarchy = cv.findContours(img_eroded, cv.RETR_TREE, cv.CHAIN_APPROX_NONE)
color_grid_contours = FilterColorGrid(contours, small_grid_size_range)
centroid_positions = []
centroid_positions = FindContourCenter(color_grid_contours)
centroids_no_overlap = []
centroids_no_overlap = FilterOutOverlapPoint(color_grid_contours,centroid_positions)
#small to big, base on y axis
centroids_no_overlap = sorted(centroids_no_overlap,key=lambda centroid: centroid[1])
rad_top = math.atan((centroids_no_overlap[1][1] - centroids_no_overlap[0][1]) /
(centroids_no_overlap[1][0] - centroids_no_overlap[0][0]))
angle_top = rad_top * 180 / PI
centroids_no_overlap_size = len(centroids_no_overlap)
rad_bottom = math.atan((centroids_no_overlap[centroids_no_overlap_size - 1][1] - centroids_no_overlap[centroids_no_overlap_size - 2][1]) /
(centroids_no_overlap[centroids_no_overlap_size - 1][0] - centroids_no_overlap[centroids_no_overlap_size - 2][0]))
angle_bottom = rad_bottom * 180 / PI
angle_subtract = abs(angle_top - angle_bottom)
if angle_subtract >= ANGLE_SUBTRACT_MAX_ENDURANCE:
orientation_left_top = centroids_no_overlap[0]
orientation_right_bottom=[0.0,0.0]
for centroid in centroids_no_overlap:
if centroid[0] < orientation_left_top[0]:
orientation_left_top[0] = centroid[0]
if centroid[1] < orientation_left_top[1]:
orientation_left_top[1] = centroid[1]
if centroid[0] > orientation_right_bottom[0]:
orientation_right_bottom[0] = centroid[0]
if centroid[1] > orientation_right_bottom[1]:
orientation_right_bottom[1] = centroid[1]
translation=[0.0,0.0]
translation[0] = (orientation_right_bottom[0] - orientation_left_top[0]) / (TOTAL_COLUMNS - 1)
translation[1] = (orientation_right_bottom[1] - orientation_left_top[1]) / (TOTAL_ROWS - 1)
grids_position = np.zeros((TOTAL_ROWS, TOTAL_COLUMNS, 2), dtype=np.int)
row_count = 0
col_count = 0
while(row_count < TOTAL_ROWS):
col_count = 0
while(col_count < TOTAL_COLUMNS):
grids_position[row_count][col_count][0] = orientation_left_top[0] + translation[0]*col_count
grids_position[row_count][col_count][1] = orientation_left_top[1] + translation[1]*row_count
col_count += 1
row_count += 1
return grids_position
else:
angle_avg = (angle_top + angle_bottom) / 2
img_rotation = np.zeros(img_gray.shape, dtype=np.uint8)
rows, cols = img_gray.shape
center = [cols / 2, rows / 2]
rotation_mat = cv.getRotationMatrix2D((center[0],center[1]),angle_avg,1.0)
img_rotation = cv.warpAffine(img_eroded,rotation_mat,(cols,rows))
contours, _hierarchy = cv.findContours(img_rotation, cv.RETR_TREE, cv.CHAIN_APPROX_NONE)
color_grid_contours = FilterColorGrid(contours, small_grid_size_range)
centroid_positions = []
centroid_positions = FindContourCenter(color_grid_contours)
orientation_left_top = centroid_positions[0]
orientation_right_bottom=[0.0,0.0]
# contour_arr_all = np.zeros(img_gray.shape, dtype=np.uint8)
i=0
for centroid in centroid_positions:
if centroid[0] < orientation_left_top[0]:
orientation_left_top[0] = centroid[0]
if centroid[1] < orientation_left_top[1]:
orientation_left_top[1] = centroid[1]
if centroid[0] > orientation_right_bottom[0]:
orientation_right_bottom[0] = centroid[0]
if centroid[1] > orientation_right_bottom[1]:
orientation_right_bottom[1] = centroid[1]
i+=1
translation=[0.0,0.0]
translation[0] = (orientation_right_bottom[0] - orientation_left_top[0]) / (TOTAL_COLUMNS - 1)
translation[1] = (orientation_right_bottom[1] - orientation_left_top[1]) / (TOTAL_ROWS - 1)
grid_coordinate = np.zeros((TOTAL_ROWS, TOTAL_COLUMNS, 2), dtype=np.int)
row_count = 0
col_count = 0
while(row_count < TOTAL_ROWS):
col_count = 0
while(col_count < TOTAL_COLUMNS):
grid_coordinate[row_count][col_count][0] = orientation_left_top[0] + translation[0]*col_count
grid_coordinate[row_count][col_count][1] = orientation_left_top[1] + translation[1]*row_count
col_count += 1
row_count +=1
grids_position = np.zeros((TOTAL_ROWS, TOTAL_COLUMNS, 2), dtype=np.int)
temp_coordinate = [0,0]
row_count = 0
col_count = 0
while(row_count < TOTAL_ROWS):
col_count = 0
while(col_count < TOTAL_COLUMNS):
temp_coordinate = GetRotationPoint((grid_coordinate[row_count][col_count][0],grid_coordinate[row_count][col_count][1]), cols, rows, -1 * angle_avg)
grids_position[row_count][col_count][0] = temp_coordinate[0]
grids_position[row_count][col_count][1] = temp_coordinate[1]
col_count += 1
row_count += 1
return grids_position
def main(filepath):
#-------------------------------------------------------------------------------
# Image Preprocessing (Blurring, Noise Removal, Binarization, Deskewing)
#-------------------------------------------------------------------------------
# Noise Removal: https://docs.opencv.org/3.3.1/d5/d69/tutorial_py_non_local_means.html
# Deskewing: https://www.pyimagesearch.com/2017/02/20/text-skew-correction-opencv-python/
# Binarization + Blurring (Otsu): https://docs.opencv.org/3.3.1/d7/d4d/tutorial_py_thresholding.html
# ============ Read Image ============
#img_file = sys.argv[1:][0]
#img_file = path + r'\test\mary.jpg'
img = cv2.imread(filepath, 0)
# ============ Noise Removal ============
img = cv2.fastNlMeansDenoising(img, None, 10, 7, 21)
# ============ Binarization ============
# Global Thresholding
# retval, img = cv2.threshold(img,127,255,cv2.THRESH_BINARY)
# Otsu's Thresholding
retval, img = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
#cv2.imwrite(path + r'\output\binarized.jpg', img)
# ============ Deskewing ============
# angle, img = deskew(img)
# print("[INFO] Deskew Angle: {:.3f}".format(angle))
# cv2.imshow("Input", img)
# cv2.waitKey(0)
# ============ Reference Lengths ============
# Reference lengths staff line thickness (staffline_height)
# and vertical line distance within the same staff (staffspace_height)
# computed, providing the basic scale for relative size comparisons
# Use run-length encoding on columns to estimate staffline height and staffspace height
line_width, line_spacing = get_ref_lengths(img)
#-------------------------------------------------------------------------------
# Staff Line Detection
#-------------------------------------------------------------------------------
# In practice, several horizontal projections on images with slightly different
# rotation angles are computed to deal with not completely horizontal staff lines.
# The projection with the highest local maxima is then chosen.
# ============ Find Staff Line Rows ============
all_staffline_vertical_indices = find_staffline_rows(img, line_width, line_spacing)
# ============ Find Staff Line Columns ============
# Find column with largest index that has no black pixels
all_staffline_horizontal_indices = find_staffline_columns(img, all_staffline_vertical_indices, line_width, line_spacing)
# ============ Show Detected Staffs ============
staffs = []
half_dist_between_staffs = (all_staffline_vertical_indices[1][0][0] - all_staffline_vertical_indices[0][4][line_width - 1])//2
for i in range(len(all_staffline_vertical_indices)):
# Create Bounding Box
x = all_staffline_horizontal_indices[i][0]
y = all_staffline_vertical_indices[i][0][0]
width = all_staffline_horizontal_indices[i][1] - x
height = all_staffline_vertical_indices[i][4][line_width - 1] - y
staff_box = BoundingBox(x, y, width, height)
# Create Cropped Staff Image
staff_img = img[max(0, y - half_dist_between_staffs): min(y+ height + half_dist_between_staffs, img.shape[0] - 1), x:x+width]
# Normalize Staff line Numbers to Cropped Image
pixel = half_dist_between_staffs
normalized_staff_line_vertical_indices = []
for j in range(5):
line = []
for k in range(line_width):
line.append(pixel)
pixel += 1
normalized_staff_line_vertical_indices.append(line)
pixel += line_spacing + 1
staff = Staff(normalized_staff_line_vertical_indices, staff_box, line_width, line_spacing, staff_img)
staffs.append(staff)
staff_boxes_img = img.copy()
staff_boxes_img = cv2.cvtColor(staff_boxes_img, cv2.COLOR_GRAY2RGB)
red = (0, 0, 255)
box_thickness = 2
for staff in staffs:
box = staff.getBox()
box.draw(staff_boxes_img, red, box_thickness)
x = int(box.getCorner()[0] + (box.getWidth() // 2))
y = int(box.getCorner()[1] + box.getHeight() + 35)
cv2.putText(staff_boxes_img, "Staff", (x, y), cv2.FONT_HERSHEY_DUPLEX, 0.9 , red)
#cv2.imwrite(path + r'\output\detected_staffs.jpg', staff_boxes_img)
# open_file('output/detected_staffs.jpg')
#-------------------------------------------------------------------------------
# Symbol Segmentation, Object Recognition, and Semantic Reconstruction
#-------------------------------------------------------------------------------
# The score is then divided into regions of interest to localize and isolate the musical primitives.
# Music score is analyzed and split by staves
# Primitive symbols extracted
# Find all primitives on each stave first
# then move from left to right and create structure
# ============ Determine Clef, Time Signature ============
staff_imgs_color = []
for i in range(len(staffs)):
red = (0, 0, 255)
box_thickness = 2
staff_img = staffs[i].getImage()
staff_img_color = staff_img.copy()
staff_img_color = cv2.cvtColor(staff_img_color, cv2.COLOR_GRAY2RGB)
# ------- Clef -------
for clef in clef_imgs:
clef_boxes = locate_templates(staff_img, clef_imgs[clef], clef_lower, clef_upper, clef_thresh)
clef_boxes = merge_boxes([j for i in clef_boxes for j in i], 0.5)
if (len(clef_boxes) == 1):
staffs[i].setClef(clef)
clef_boxes_img = staffs[i].getImage()
clef_boxes_img = clef_boxes_img.copy()
for boxes in clef_boxes:
boxes.draw(staff_img_color, red, box_thickness)
x = int(boxes.getCorner()[0] + (boxes.getWidth() // 2))
y = int(boxes.getCorner()[1] + boxes.getHeight() + 10)
cv2.putText(staff_img_color, "{} clef".format(clef), (x, y), cv2.FONT_HERSHEY_DUPLEX, 0.9, red)
break
# # ------- Time -------
for time in time_imgs:
time_boxes = locate_templates(staff_img, time_imgs[time], time_lower, time_upper, time_thresh)
time_boxes = merge_boxes([j for i in time_boxes for j in i], 0.5)
if (len(time_boxes) == 1):
staffs[i].setTimeSignature(time)
for boxes in time_boxes:
boxes.draw(staff_img_color, red, box_thickness)
x = int(boxes.getCorner()[0] - (boxes.getWidth() // 2))
y = int(boxes.getCorner()[1] + boxes.getHeight() + 20)
cv2.putText(staff_img_color, "{} time".format(time), (x, y), cv2.FONT_HERSHEY_DUPLEX, 0.9, red)
break
elif (len(time_boxes) == 0 and i > 0):
# Take time signature of previous staff
previousTime = staffs[i-1].getTimeSignature()
staffs[i].setTimeSignature(previousTime)
#print("[INFO] No time signature found on staff", i + 1, ". Using time signature from previous staff line: ", previousTime)
break
staff_imgs_color.append(staff_img_color)
# ============ Find Primitives ============
# always assert that notes in a bar equal duration dictated by time signature
for i in range(len(staffs)):
staff_primitives = []
staff_img = staffs[i].getImage()
staff_img_color = staff_imgs_color[i]
red = (0, 0, 255)
box_thickness = 2
# ------- Find primitives on staff -------
sharp_boxes = locate_templates(staff_img, sharp_imgs, sharp_lower, sharp_upper, sharp_thresh)
sharp_boxes = merge_boxes([j for i in sharp_boxes for j in i], 0.5)
for box in sharp_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "sharp"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
sharp = Primitive("sharp", 0, box)
staff_primitives.append(sharp)
flat_boxes = locate_templates(staff_img, flat_imgs, flat_lower, flat_upper, flat_thresh)
flat_boxes = merge_boxes([j for i in flat_boxes for j in i], 0.5)
for box in flat_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "flat"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
flat = Primitive("flat", 0, box)
staff_primitives.append(flat)
quarter_boxes = locate_templates(staff_img, quarter_note_imgs, quarter_note_lower, quarter_note_upper, quarter_note_thresh)
quarter_boxes = merge_boxes([j for i in quarter_boxes for j in i], 0.5)
for box in quarter_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "1/4 note"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
pitch = staffs[i].getPitch(round(box.getCenter()[1]))
quarter = Primitive("note", 1, box, pitch)
staff_primitives.append(quarter)
half_boxes = locate_templates(staff_img, half_note_imgs, half_note_lower, half_note_upper, half_note_thresh)
half_boxes = merge_boxes([j for i in half_boxes for j in i], 0.5)
for box in half_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "1/2 note"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
pitch = staffs[i].getPitch(round(box.getCenter()[1]))
half = Primitive("note", 2, box, pitch)
staff_primitives.append(half)
whole_boxes = locate_templates(staff_img, whole_note_imgs, whole_note_lower, whole_note_upper, whole_note_thresh)
whole_boxes = merge_boxes([j for i in whole_boxes for j in i], 0.5)
for box in whole_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "1 note"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
pitch = staffs[i].getPitch(round(box.getCenter()[1]))
whole = Primitive("note", 4, box, pitch)
staff_primitives.append(whole)
eighth_boxes = locate_templates(staff_img, eighth_rest_imgs, eighth_rest_lower, eighth_rest_upper, eighth_rest_thresh)
eighth_boxes = merge_boxes([j for i in eighth_boxes for j in i], 0.5)
for box in eighth_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "1/8 rest"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
eighth = Primitive("rest", 0.5, box)
staff_primitives.append(eighth)
quarter_boxes = locate_templates(staff_img, quarter_rest_imgs, quarter_rest_lower, quarter_rest_upper, quarter_rest_thresh)
quarter_boxes = merge_boxes([j for i in quarter_boxes for j in i], 0.5)
for box in quarter_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "1/4 rest"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
quarter = Primitive("rest", 1, box)
staff_primitives.append(quarter)
half_boxes = locate_templates(staff_img, half_rest_imgs, half_rest_lower, half_rest_upper, half_rest_thresh)
half_boxes = merge_boxes([j for i in half_boxes for j in i], 0.5)
for box in half_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "1/2 rest"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
half = Primitive("rest", 2, box)
staff_primitives.append(half)
whole_boxes = locate_templates(staff_img, whole_rest_imgs, whole_rest_lower, whole_rest_upper, whole_rest_thresh)
whole_boxes = merge_boxes([j for i in whole_boxes for j in i], 0.5)
for box in whole_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "1 rest"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
whole = Primitive("rest", 4, box)
staff_primitives.append(whole)
flag_boxes = locate_templates(staff_img, eighth_flag_imgs, eighth_flag_lower, eighth_flag_upper, eighth_flag_thresh)
flag_boxes = merge_boxes([j for i in flag_boxes for j in i], 0.5)
for box in flag_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "1/8 flag"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
flag = Primitive("eighth_flag", 0, box)
staff_primitives.append(flag)
bar_boxes = locate_templates(staff_img, bar_imgs, bar_lower, bar_upper, bar_thresh)
bar_boxes = merge_boxes([j for i in bar_boxes for j in i], 0.5)
for box in bar_boxes:
box.draw(staff_img_color, red, box_thickness)
text = "line"
font = cv2.FONT_HERSHEY_DUPLEX
textsize = cv2.getTextSize(text, font, fontScale=0.7, thickness=1)[0]
x = int(box.getCorner()[0] - (textsize[0] // 2))
y = int(box.getCorner()[1] + box.getHeight() + 20)
cv2.putText(staff_img_color, text, (x, y), font, fontScale=0.7, color=red, thickness=1)
line = Primitive("line", 0, box)
staff_primitives.append(line)
#cv2.imwrite(path + r'\output\staff_{}_primitives.jpg'.format(i+1), staff_img_color)
# open_file("output/staff_{}_primitives.jpg".format(i+1))
# ------- Sort primitives on staff from left to right -------
staff_primitives.sort(key=lambda primitive: primitive.getBox().getCenter())
eighth_flag_indices = []
for j in range(len(staff_primitives)):
if (staff_primitives[j].getPrimitive() == "eighth_flag"):
# Find all eighth flags
eighth_flag_indices.append(j)
# ------- Correct for eighth notes -------
# Sort out eighth flags
# Assign to closest note
for j in eighth_flag_indices:
distances = []
distance = staff_primitives[j].getBox().distance(staff_primitives[j-1].getBox())
distances.append(distance)
if (j + 1 < len(staff_primitives)):
distance = staff_primitives[j].getBox().distance(staff_primitives[j+1].getBox())
distances.append(distance)
if (distances[1] and distances[0] > distances[1]):
staff_primitives[j+1].setDuration(0.5)
else:
staff_primitives[j-1].setDuration(0.5)
del staff_primitives[j]
# Correct for beamed eighth notes
# If number of pixels in center row of two notes
# greater than 5 * line_width, then notes are
# beamed
for j in range(len(staff_primitives)):
if (j+1 < len(staff_primitives)
and staff_primitives[j].getPrimitive() == "note"
and staff_primitives[j+1].getPrimitive() == "note"
and (staff_primitives[j].getDuration() == 1 or staff_primitives[j].getDuration() == 0.5)
and staff_primitives[j+1].getDuration() == 1):
# Notes of interest
note_1_center_x = staff_primitives[j].getBox().getCenter()[0]
note_2_center_x = staff_primitives[j+1].getBox().getCenter()[0]
# Regular number of black pixels in staff column
num_black_pixels = 5 * staffs[i].getLineWidth()
# Actual number of black pixels in mid column
center_column = (note_2_center_x - note_1_center_x) // 2
mid_col = staff_img[:, int(note_1_center_x + center_column)]
num_black_pixels_mid = len(np.where(mid_col == 0)[0])
if (num_black_pixels_mid > num_black_pixels):
# Notes beamed
# Make eighth note length
staff_primitives[j].setDuration(0.5)
staff_primitives[j+1].setDuration(0.5)
# ------- Account for Key Signature -------
num_sharps = 0
num_flats = 0
j = 0
while (staff_primitives[j].getDuration() == 0):
accidental = staff_primitives[j].getPrimitive()
if (accidental == "sharp"):
num_sharps += 1
j += 1
elif (accidental == "flat"):
num_flats += 1
j += 1
# Check if last accidental belongs to note
if (j != 0):
# Determine if accidental coupled with first note
# Center of accidental should be within a note width from note
max_accidental_offset_x = staff_primitives[j].getBox().getCenter()[0] - staff_primitives[j].getBox().getWidth()
accidental_center_x = staff_primitives[j-1].getBox().getCenter()[0]
accidental_type = staff_primitives[j-1].getPrimitive()
if (accidental_center_x > max_accidental_offset_x):
num_sharps = num_sharps - 1 if accidental_type == "sharp" else num_sharps
num_flats = num_flats - 1 if accidental_type == "flat" else num_flats
# Modify notes in staff
notes_to_modify = []
if (accidental_type == "sharp"):
notes_to_modify = key_signature_changes[accidental_type][num_sharps]
# Remove accidentals from primitive list
staff_primitives = staff_primitives[num_sharps:]
else:
notes_to_modify = key_signature_changes[accidental_type][num_flats]
# Remove accidentals from primitive list
staff_primitives = staff_primitives[num_flats:]
for primitive in staff_primitives:
type = primitive.getPrimitive()
note = primitive.getPitch()
if (type == "note" and note[0] in notes_to_modify):
new_note = MIDI_to_pitch[pitch_to_MIDI[note] + 1] if accidental_type == "sharp" else MIDI_to_pitch[pitch_to_MIDI[note] - 1]
primitive.setPitch(new_note)
# ------- Apply Sharps and Flats -------
primitive_indices_to_remove = []
for j in range(len(staff_primitives)):
accidental_type = staff_primitives[j].getPrimitive()
if (accidental_type == "flat" or accidental_type == "sharp"):
max_accidental_offset_x = staff_primitives[j+1].getBox().getCenter()[0] - staff_primitives[j+1].getBox().getWidth()
accidental_center_x = staff_primitives[j].getBox().getCenter()[0]
primitive_type = staff_primitives[j+1].getPrimitive()
if (accidental_center_x > max_accidental_offset_x and primitive_type == "note"):
note = staff_primitives[j+1].getPitch()
new_note = MIDI_to_pitch[pitch_to_MIDI[note] + 1] if accidental_type == "sharp" else MIDI_to_pitch[pitch_to_MIDI[note] - 1]
staff_primitives[j+1].setPitch(new_note)
primitive_indices_to_remove.append(i)
# Removed actioned accidentals
for j in primitive_indices_to_remove:
del staff_primitives[j]
# ------- Assemble Staff -------
bar = Bar()
while (len(staff_primitives) > 0):
primitive = staff_primitives.pop(0)
if (primitive.getPrimitive() != "line"):
bar.addPrimitive(primitive)
else:
staffs[i].addBar(bar)
bar = Bar()
# Add final bar in staff
staffs[i].addBar(bar)
# -------------------------------------------------------------------------------
# Output MusicXml
# -------------------------------------------------------------------------------
score = Score()
part = Part("Piano")
score.append(part)
measures = []
time = int(staffs[0].getTimeSignature())
clef = staffs[0].getClef()
key_signature = "C"
if(num_sharps!=0):
key_signature = key[num_sharps - 1]
if(num_flats!=0):
key_signature = key[len(key) - num_flats]
for i in range(len(staffs)):
bars = staffs[i].getBars()
for j in range(len(bars)):
m = Measure(clef=clef, time_signature=(time//10, time%10) if i == 0 else None)
primitives = bars[j].getPrimitives()
for k in range(len(primitives)):
duration = primitives[k].getDuration()
if (primitives[k].getPrimitive() == "note"):
pitch = primitives[k].getPitch()
m.append(Note(pitch, duration))
elif(primitives[k].getPrimitive() == "rest"):
m.append(Rest(duration))
measures.append(m)
part.extend(measures)
# ------- Write to disk -------
print("[INFO] Writing MusicXml to disk")
#outputpath = path + '\\output\\' + key_signature + 'major.xml'
output_path = "/sdcard/Documents/" + key_signature + "sheet.xml"
score.export_to_file(output_path)
return output_path
def noise(self, h=20):
"""降噪"""
self.image = cv2.fastNlMeansDenoising(self.image, h=h)
def denoise(img):
return cv2.fastNlMeansDenoising(img, None, 10, 7, 21)
def denoised(image: np.array) -> np.array:
"""Removes noise from the image."""
if is_colored(image):
return cv2.fastNlMeansDenoisingColored(image)
else:
return cv2.fastNlMeansDenoising(image)
