def apply_single(self, data: ImageOperationData):
image = data.images[0].image
marked_regions = np.zeros(image.shape, dtype=np.uint8)
i = 1
text_blocks = [tr for tr in data.page.blocks_of_type(self.text_region_types)]
def p2i(p):
return data.page.page_to_image_scale(p, data.scale_reference)
for tr in text_blocks:
for tl in tr.lines:
p2i(tl.coords).draw(marked_regions, i, 0, fill=True)
i += 1
out = []
i = 1
for tr in text_blocks:
for tl in tr.lines:
mask = marked_regions == i
if len(tl.text()) == 0:
continue
if np.sum(mask) == 0: # empty mask, skip
continue
else:
img_data = copy(data)
img_data.page_image = image
img_data.text_line = tl
img_data.images = [ImageData(mask, True), ImageData(image, True)]
cropped = self.cropper.apply_single(img_data)[0]
self._extract_image_op(img_data)
img_data.params = (i, cropped.params)
out.append(img_data)
i += 1
return out
def apply_single(self, data: ImageOperationData):
image = data.images[0].image
all_mls = data.page.all_music_lines()
all_tls = data.page.all_text_lines()
def extract_transformed_coords(ml: Line) -> List[Coords]:
lines = ml.staff_lines.sorted()
return [data.page.page_to_image_scale(sl.coords, data.scale_reference) for sl in lines]
def transform_point(p) -> np.array:
return data.page.page_to_image_scale(p, data.scale_reference)
s = [extract_transformed_coords(ml) for ml in all_mls]
# dewarp
images = [Image.fromarray(image)]
dewarper = Dewarper(images[0].size, s)
dew_page, = tuple(map(np.array, dewarper.dewarp(images)))
out = []
for tl in all_tls:
dew_coords = Coords(dewarper.transform_points(transform_point(tl.coords.points)))
aabb = dew_coords.aabb()
if aabb.area() == 0:
continue
image = dew_page[int(aabb.top()):int(aabb.bottom()), int(aabb.left()):int(aabb.right())]
img_data = copy(data)
img_data.page_image = image
img_data.text_line = tl
img_data.images = [ImageData(image, True)]
img_data.params = (dewarper, aabb)
out.append(img_data)
if False:
import matplotlib.pyplot as plt
if True:
f, ax = plt.subplots(len(out), 1)
for o, a in zip(out, ax):
a.imshow(o.images[0].image)
plt.show()
else:
f, ax = plt.subplots(1, 2)
ax[0].imshow(image)
ax[1].imshow(dew_page)
plt.show()
return out
def _extract_image_op(self, data: ImageOperationData):
data.images = [
data.images[1],
ImageData(data.images[0].image * data.images[1].image, False) if self.extract_region_only else data.images[1]
]
def apply_single(self, data: ImageOperationData):
image = data.images[0].image
marked_regions = np.zeros(image.shape, dtype=np.uint8)
marked_staff_lines = np.zeros(image.shape, dtype=np.uint8)
page = data.page
def scale(p):
return page.page_to_image_scale(p, data.scale_reference)
i = 1
s = []
for mr in data.page.music_blocks():
for ml in mr.lines:
s.append(ml)
scale(ml.coords).draw(marked_regions, i, 0, fill=True)
for sl in ml.staff_lines:
scale(sl.coords).draw(marked_staff_lines,
color=1,
thickness=self.gt_line_thickness)
i += 1
out = []
if self.full_page:
image_data = copy(data)
image_data.images = [
ImageData(marked_regions, True),
ImageData(image, False),
ImageData(marked_staff_lines, True)
]
image_data.params = None
image_data.page_image = image
image_data.music_lines = s
out.append(image_data)
else:
i = 1
for mr in data.page.music_regions:
for ml in mr.staffs:
mask = marked_regions == i
if np.sum(mask) == 0: # empty mask, skip
continue
else:
img_data = copy(data)
img_data.page_image = image
img_data.music_region = mr
img_data.music_line = ml
img_data.music_lines = [ml]
img_data.images = [
ImageData(mask, True),
ImageData(image, False),
ImageData(marked_staff_lines, True)
]
cropped = self.cropper.apply_single(img_data)[0]
self._extract_image_op(img_data)
img_data.params = (i, cropped.params)
out.append(img_data)
i += 1
return out
def _extract_image_op(self, data: ImageOperationData):
data.images = [
data.images[1],
ImageData(data.images[0].image * data.images[1].image, False)
] + data.images[2:]
def apply_single(self,
data: ImageOperationData,
debug=False) -> OperationOutput:
image = data.images[0].image
labels = np.zeros(image.shape, dtype=np.uint8)
marked_symbols = np.zeros(labels.shape, dtype=np.uint8)
i = 1
s: List[List[Coords]] = []
def extract_transformed_coords(ml: Line) -> List[Coords]:
lines = ml.staff_lines.sorted()
return [
data.page.page_to_image_scale(sl.coords, data.scale_reference)
for sl in lines
]
for mr in data.page.music_blocks():
for ml in mr.lines:
coords = extract_transformed_coords(ml)
s.append(coords)
if self.staff_lines_only:
# draw staff lines instead of full area, however add an average line distance to top and bottom
avg_d = data.page.page_to_image_scale(
ml.avg_line_distance(), data.scale_reference)
top = max(0, int(coords[0].points[:, 1].min() - avg_d))
bot = min(labels.shape[0],
int(coords[-1].points[:, 1].max() + avg_d))
left = max(0, int(coords[0].points[:, 0].min()))
right = min(labels.shape[1],
int(coords[0].points[:, 0].max()))
labels[top:bot, left:right] = i
else:
data.page.page_to_image_scale(
ml.coords, data.scale_reference).draw(labels,
i,
0,
fill=True)
self._symbols_to_mask(ml, marked_symbols, data.page,
data.scale_reference)
i += 1
if self.dewarp:
images = [
Image.fromarray(image),
Image.fromarray(labels),
Image.fromarray(marked_symbols)
]
dewarper = Dewarper(images[0].size, s)
dew_page, dew_labels, dew_symbols = tuple(
map(np.array, dewarper.dewarp(images, None)))
else:
dewarper = None
dew_page, dew_labels, dew_symbols = image, labels, marked_symbols
if debug:
import matplotlib.pyplot as plt
f, ax = plt.subplots(1, 2)
ax[0].imshow(labels)
ax[1].imshow(dew_labels)
plt.show()
i = 1
out = []
for mr in data.page.music_blocks():
for ml in mr.lines:
mask = dew_labels == i
if np.sum(mask) != 0: # empty mask, skip
img_data = copy(data)
img_data.page_image = image
img_data.music_region = mr
img_data.music_line = ml
img_data.images = [
ImageData(mask, True),
ImageData(dew_page, False),
ImageData(dew_symbols, True)
]
cropped = self.cropper.apply_single(img_data)[0]
self._extract_image_op(img_data)
if self.center:
coords = extract_transformed_coords(ml)
r = self._resize_to_height(cropped.images,
coords,
rect=cropped.params)
if r is not None: # Invalid resize (probably no staff lines present)
img_data.images, r_params = r
img_data.params = (i, cropped.params, r_params, s,
dewarper)
out.append(img_data)
else:
img_data.params = (i, cropped.params, (0, ), s,
dewarper)
out.append(img_data)
i += 1
if debug:
import matplotlib.pyplot as plt
f, ax = plt.subplots(len(out), 3)
for i, o in enumerate(out):
ax[i, 0].imshow(o.images[0].image)
ax[i, 1].imshow(o.images[1].image)
ax[i, 2].imshow(o.images[2].image)
plt.show()
return out