def test_norm_poly_dists(self):
des_dist = 2
# Should not be changed (n_points <= 20 after blow_up)
polygon1 = Polygon(list(range(20)), list(range(20)), 20)
res1 = polygon1
# Should be changed, s.t. the normed polygon has 20 (nearly equidistant) pixels
polygon2 = Polygon(list(range(30)), list(range(30)), 30)
res2 = Polygon([
0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, 16, 18, 19, 21, 22, 24, 25,
27, 29
], [
0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, 16, 18, 19, 21, 22, 24, 25,
27, 29
], 20)
# Should not be changed, since des_dist = 2
polygon3 = Polygon(list(range(0, 40, 2)), list(range(0, 40, 2)), 20)
res3 = polygon3
# Should be changed, s.t. every two pixels have a distance of des_dist = 2 (except for the last two pixels)
polygon4 = Polygon(list(range(0, 90, 3)), list(range(0, 90, 3)), 30)
res4 = Polygon(
list(range(0, 86, 2)) + [87],
list(range(0, 86, 2)) + [87], 44)
poly_list = [polygon1, polygon2, polygon3, polygon4]
normed_poly_list = polygon.norm_poly_dists(poly_list, des_dist)
res_list = [res1, res2, res3, res4]
for normed_poly, res in zip(normed_poly_list, res_list):
self.assertEqual(res.x_points, normed_poly.x_points)
self.assertEqual(res.y_points, normed_poly.y_points)
self.assertEqual(res.n_points, normed_poly.n_points)
def calc_measure_for_page_baseline_polys(self,
polys_truth,
polys_reco,
use_java_code=True):
""" Calculates the BaselinMeasure stats for given truth and reco polygons of a single page and adds the results
to the BaselineMeasure structure.
NOTE: We can choose between the usage of java (much more faster!!!) or python methods.
:param polys_truth: list of TRUTH polygons corresponding to a single page
:param polys_reco: list of RECO polygons corresponding to a single page
:param use_java_code: usage of methods written in java or not
"""
assert type(polys_truth) == list and type(
polys_reco) == list, "polys_truth and polys_reco have to be lists"
assert all([isinstance(poly, Polygon) for poly in polys_truth + polys_reco]), \
"elements of polys_truth and polys_reco have to be Polygons"
# call java code to execute the method
if use_java_code:
polys_truth_java = []
polys_reco_java = []
for poly in polys_truth:
polys_truth_java.append(
jpype.java.awt.Polygon(poly.x_points, poly.y_points,
poly.n_points))
for poly in polys_reco:
polys_reco_java.append(
jpype.java.awt.Polygon(poly.x_points, poly.y_points,
poly.n_points))
java_object = jpype.JPackage(
"citlab_article_separation_measure.external.java").Util()
pr_list = \
java_object.calcMetricForPageBaseLinePolys(polys_truth_java, polys_reco_java,
self.max_tols.tolist(), self.poly_tick_dist, self.rel_tol)
precision = np.array(
[list(pr_list[0][i]) for i in range(len(pr_list[0]))])
recall = np.array(
[list(pr_list[1][i]) for i in range(len(pr_list[0]))])
# call python code to execute the method
else:
# Normalize baselines, so that poly points have a desired "distance"
polys_truth_norm = norm_poly_dists(polys_truth,
self.poly_tick_dist)
polys_reco_norm = norm_poly_dists(polys_reco, self.poly_tick_dist)
# Optionally calculate tolerances
if self.max_tols[0] < 0:
# call python class to calculate the tolerances
tols = calc_tols(polys_truth_norm, self.poly_tick_dist, 250,
self.rel_tol)
self.truth_line_tols = np.expand_dims(tols, axis=1)
else:
self.truth_line_tols = np.tile(
self.max_tols, [len(polys_truth_norm), 1]).astype(float)
# For each reco poly calculate the precision values for all tolerances
precision = self.calc_precision(polys_truth_norm, polys_reco_norm)
# For each truth_poly calculate the recall values for all tolerances
recall = self.calc_recall(polys_truth_norm, polys_reco_norm)
# add results
self.measure.add_per_dist_tol_tick_per_line_precision(precision)
self.measure.add_per_dist_tol_tick_per_line_recall(recall)
self.truth_line_tols = None
def smooth_surrounding_polygon(polygon,
poly_norm_dist=10,
orientation_dims=(400, 800, 600, 400),
offset=0):
"""
Takes a "crooked" polygon and smooths it, by approximating vertical and horizontal edges.
1.) The polygon gets normalized, where the resulting vertices are at most `poly_norm_dist` pixels apart.
2.) For each vertex of the original polygon an orientation is determined:
2.1) Four cones (North, East, South, West) are generated, with the dimensions given by `or_dims`
(width_vertical, height_vertical, width_horizontal, height_horizontal), i.e. North and South rectangles
have dimensions width_v x height_v, whereas East and West rectangles have dimensions width_h x height_h.
2.2) The offset controls how far the cones overlap (e.g. how far the north cone gets translated south)
2.3) Each rectangle counts the number of contained points from the normalized polygon
2.4) The top two rectangle counts determine the orientation of the vertex: vertical, horizontal or one
of the four possible corner types.
3.) Vertices with a differing orientation to its agreeing neighbours are assumed to be mislabeled and
get its orientation converted to its neighbours.
4.) Corner clusters of the same type need to be shrunken down to one corner, with the rest being converted
to verticals / horizontals.
5.) Clusters between corners (corner-V/H-...-V/H-corner) get smoothed if they contain at least five points,
by taking the average over the y-coordinates for horizontal edges and the average over the x-coordinates for
vertical edges.
:param polygon: (not necessarily closed) polygon, represented as a list of tuples (x,y)
:param poly_norm_dist: int, distance between pixels in normalized polygon
:param orientation_dims: tuple (width_v, height_v, width_h, height_h), the dimensions of the orientation rectangles
:param offset: int, number of pixel that the orientation cones overlap
:return: dict (keys = article_id, values = smoothed polygons)
"""
if isinstance(polygon, Polygon):
polygon = polygon.as_list()
# Normalize polygon points over surrounding polygon
surrounding_polygon = polygon.copy()
# Optionally close polygon
if surrounding_polygon[0] != surrounding_polygon[-1]:
surrounding_polygon.append(polygon[0])
# print("--------------------------")
# Normalize polygon points
poly_xs, poly_ys = zip(*surrounding_polygon)
poly = Polygon(list(poly_xs), list(poly_ys), len(poly_xs))
poly_norm = norm_poly_dists([poly], des_dist=poly_norm_dist)[0]
# Get polygon dimensions
poly_bb = poly.get_bounding_box()
poly_h, poly_w = poly_bb.height, poly_bb.width
# Build final dimensions for orientation objects
dims_flex = [poly_h // 2, poly_h // 2, poly_w // 2, poly_h // 3]
dims_min = [100, 80, 100, 60]
dims = [
max(min(x, y), z)
for x, y, z in zip(orientation_dims, dims_flex, dims_min)
]
# print("poly_height {}, poly_width {}".format(poly_h, poly_w))
# print("orientation_dims ", orientation_dims)
# print("dims_flex ", dims_flex)
# print("dims ", dims)
# Determine orientation for every vertex of the (original) polygon
oriented_points = []
for pt in polygon:
# Build up 4 cones in each direction (N, E, S, W)
cones = get_orientation_cones(pt, dims, offset)
# Count the number of contained points from the normalized polygon in each cone
points_in_cones = {'n': 0, 'e': 0, 's': 0, 'w': 0}
for o in cones:
for pn in zip(poly_norm.x_points, poly_norm.y_points):
if cones[o].contains_point(pn):
points_in_cones[o] += 1
# Get orientation of vertex by top two counts
sorted_counts = sorted(points_in_cones.items(),
key=lambda kv: kv[1],
reverse=True)
top_two = list(zip(*sorted_counts))[0][:2]
if 'n' in top_two and 's' in top_two:
pt_o = 'vertical'
elif 'e' in top_two and 'w' in top_two:
pt_o = 'horizontal'
elif 'e' in top_two and 's' in top_two:
pt_o = 'corner_ul'
elif 'w' in top_two and 's' in top_two:
pt_o = 'corner_ur'
elif 'w' in top_two and 'n' in top_two:
pt_o = 'corner_dr'
else:
pt_o = 'corner_dl'
# Append point and its orientation as a tuple
oriented_points.append((pt, pt_o))
# print("Type: {}, Counts: {}".format(pt_o, sorted_counts))
# Fix wrongly classified points between two same classified ones
for i in range(len(oriented_points)):
if oriented_points[i - 1][1] != oriented_points[i][1] \
and oriented_points[i - 1][1] == oriented_points[(i + 1) % len(oriented_points)][1] \
and 'corner' not in oriented_points[i - 1][1]:
oriented_points[i] = (oriented_points[i][0],
oriented_points[i - 1][1])
# Search for corner clusters of the same type and keep only one corner
# TODO: Do we need to rearrange the list to start with a corner here already?
# TODO: E.g. what if one of the clusters wraps around?
for i in range(len(oriented_points)):
# Found a corner
if 'corner' in oriented_points[i][1]:
# Get cluster (and IDs) with same corner type
corner_cluster = [(i, oriented_points[i])]
j = (i + 1) % len(oriented_points)
while oriented_points[i][1] == oriented_points[j][1]:
corner_cluster.append((j, oriented_points[j]))
j = (j + 1) % len(oriented_points)
if len(corner_cluster) > 1:
# Keep corner based on type
if 'ul' in oriented_points[i][1]:
cluster_sorted = sort_cluster_by_y_then_x(corner_cluster)
elif 'ur' in oriented_points[i][1]:
cluster_sorted = sort_cluster_by_y_then_x(corner_cluster,
inverse_x=True)
elif 'dl' in oriented_points[i][1]:
cluster_sorted = sort_cluster_by_y_then_x(corner_cluster,
inverse_y=True)
else:
cluster_sorted = sort_cluster_by_y_then_x(corner_cluster,
inverse_y=True,
inverse_x=True)
cluster_to_remove = cluster_sorted[1:]
# Convert cluster to verticals (we don't care about the type of edge vertex later on)
for c in cluster_to_remove:
idx = c[0]
oriented_points[idx] = (oriented_points[idx][0],
'vertical')
# Rearrange oriented_points list to start with a corner and wrap it around
corner_idx = 0
for i, op in enumerate(oriented_points):
if 'corner' in op[1]:
# print("Rearrange corner at index", i)
corner_idx = i
break
oriented_points = oriented_points[
corner_idx:] + oriented_points[:corner_idx]
oriented_points.append(oriented_points[0])
# print("oriented points, ", oriented_points)
# Go through the polygon and and get all corner IDs
corner_ids = []
for i, op in enumerate(oriented_points):
if 'corner' in op[1]:
corner_ids.append(i)
# print("corner IDs, ", corner_ids)
# Look at point clusters between neighboring corners
# Build up list of alternating x- and y-coordinates (representing rays) and build up the polygon afterwards
smoothed_edges = []
# Check if we start with a horizontal edge
# In this case, take the corresponding y-coordinate as the line/edge (otherwise x-coordinate)
start_cluster = oriented_points[corner_ids[0]:corner_ids[1] + 1]
# Look at corners, since this cluster will get approximated
if len(start_cluster) > 3:
is_horizontal = check_horizontal_edge(start_cluster[0][0],
start_cluster[-1][0])
# Look at first two points, since we take them as is
else:
is_horizontal = check_horizontal_edge(start_cluster[0][0],
start_cluster[1][0])
# j is the index for the x- or y-coordinate (horizontal = y, vertical = x)
j = int(is_horizontal)
# print("horizontal_edge_start", is_horizontal)
for i in range(len(corner_ids) - 1):
cluster = oriented_points[corner_ids[i]:corner_ids[i + 1] + 1]
# Approximate edges with at least 4 points (including corners)
if len(cluster) > 3:
# Plausi-Check if we're getting the correct type of edge (between corners)
# Else, switch it and insert missing ray beforehand
if not j == check_horizontal_edge(cluster[0][0], cluster[-1][0]):
# print("SWITCH", i)
smoothed_edges.append(cluster[0][0][j])
j = int(not j)
mean = 0
for pt in cluster:
mean += pt[0][j]
mean = round(float(mean) / len(cluster))
smoothed_edges.append(mean)
# Switch from x- to y-coordinate and vice versa
j = int(not j)
# Keep the rest as is, alternating between x- and y-coordinate for vertical / horizontal edges
else:
# Plausi-Check if we're getting the correct type of edge (between first two points)
# Else, switch it and insert missing ray beforehand
if not j == check_horizontal_edge(cluster[0][0], cluster[1][0]):
# print("SWITCH", i)
smoothed_edges.append(cluster[0][0][j])
j = int(not j)
# Exclude last point so we don't overlap in the next cluster
for pt in cluster[:-1]:
smoothed_edges.append(pt[0][j])
j = int(not j)
# print("smoothed_edges", smoothed_edges)
# At last step, we may need to add another ray, if the edges between last & first don't match
if i == len(corner_ids) - 2:
if j != is_horizontal:
smoothed_edges.append(cluster[-1][0][j])
# print("smoothed_edges after last step\n", smoothed_edges)
# Go over list of x-y values and build up the polygon by taking the intersection of the rays as vertices
smoothed_polygon = Polygon()
for i in range(len(smoothed_edges)):
if is_horizontal:
smoothed_polygon.add_point(
smoothed_edges[(i + 1) % len(smoothed_edges)],
smoothed_edges[i])
is_horizontal = int(not is_horizontal)
else:
smoothed_polygon.add_point(
smoothed_edges[i],
smoothed_edges[(i + 1) % len(smoothed_edges)])
is_horizontal = int(not is_horizontal)
# print("polygon", smoothed_polygon)
return smoothed_polygon
def get_data_from_pagexml(path_to_pagexml, des_dist=50, max_d=500, use_java_code=True):
"""
:param path_to_pagexml: file path
:param des_dist: desired distance (measured in pixels) of two adjacent pixels in the normed polygons
:param max_d: maximum distance (measured in pixels) for the calculation of the interline distances
:param use_java_code: usage of methods written in java (faster than python!) or not
:return: two dictionaries: {article id: corresponding list of text lines}
{text line id: (normed polygon, interline distance)}
"""
# load the page xml file
page_file = Page(path_to_pagexml)
# get all text lines article wise
art_txtlines_dict = page_file.get_article_dict()
# get all text lines of the loaded page file
lst_of_txtlines = page_file.get_textlines()
lst_of_polygons = []
lst_of_txtlines_adjusted = []
for txtline in lst_of_txtlines:
try:
# get the baseline of the text line as polygon
baseline = txtline.baseline.to_polygon()
# baselines with less than two points will skipped
if len(baseline.x_points) == len(baseline.y_points) > 1:
lst_of_polygons.append(txtline.baseline.to_polygon())
lst_of_txtlines_adjusted.append(txtline)
except(AttributeError):
# print("'NoneType' object in PAGEXML with id {} has no attribute 'to_polygon'!\n".format(txtline.id))
continue
# normed polygons
lst_of_normed_polygons = norm_poly_dists(poly_list=lst_of_polygons, des_dist=des_dist)
# interline distances
lst_of_intdists = get_list_of_interline_distances(lst_of_polygons=lst_of_polygons, max_d=max_d,
use_java_code=use_java_code)
txtline_dict = {}
for i, txtline in enumerate(lst_of_txtlines_adjusted):
# check the surrounding polygon of the text line
if txtline.surr_p is None:
normed_polygon = lst_of_normed_polygons[i]
x_points_shifted = [x + 1 for x in normed_polygon.x_points]
# y values are shifted upwards by at least one pixel
y_shift = max(int(0.95 * lst_of_intdists[i]), 1)
y_points_shifted = [y - y_shift for y in normed_polygon.y_points]
sp_points = list(zip(normed_polygon.x_points + x_points_shifted[::-1],
normed_polygon.y_points + y_points_shifted[::-1]))
for article in art_txtlines_dict:
for reference_txtline in art_txtlines_dict[article]:
if reference_txtline.id == txtline.id:
reference_txtline.surr_p = Points(sp_points)
txtline_dict.update({txtline.id: (lst_of_normed_polygons[i], lst_of_intdists[i])})
return art_txtlines_dict, txtline_dict
def initialize_gt_generation(self, des_dist=5, max_d=50):
# Create list of tuples containing the surrounding polygon, the baseline and the article id of each textline
tl_list = []
for tl in self.textlines:
try:
tl_bl = tl.baseline.to_polygon()
tl_bl.calculate_bounds()
except AttributeError:
print(f"Textline with id {tl.id} has no baseline coordinates. Skipping...")
continue
tl_surr_poly = None
try:
tl_surr_poly = tl.surr_p.to_polygon().get_bounding_box()
except (AttributeError, TypeError):
print(f"Textline with id {tl.id} has no surrounding polygon.")
tl_list.append([tl, tl_surr_poly, tl_bl, tl.get_article_id()])
# Calculate the interline distance for each baseline
# calculation of the normed polygons (includes also the calculation of their bounding boxes)
list_of_normed_polygons = norm_poly_dists([tl[2] for tl in tl_list], des_dist=des_dist)
# call java code to calculate the interline distances
java_util = jpype.JPackage("citlab_article_separation.java").Util()
list_of_normed_polygon_java = []
for poly in list_of_normed_polygons:
list_of_normed_polygon_java.append(jpype.java.awt.Polygon(poly.x_points, poly.y_points, poly.n_points))
list_of_interline_distances_java = java_util.calcInterlineDistances(list_of_normed_polygon_java, des_dist,
max_d)
list_of_interline_distances = list(list_of_interline_distances_java)
tl_list_copy = copy.deepcopy(tl_list)
# Update the bounding boxes for the textlines
for tl_tuple, tl_interdist in zip(tl_list_copy, list_of_interline_distances):
_, _, tl_bl, _ = tl_tuple
# bounding rectangle moved up and down
height_shift = int(tl_interdist)
tl_bl.bounds.translate(dx=0, dy=-height_shift)
tl_bl.bounds.height += int(1.1 * height_shift)
tl_surr_poly_final = []
has_intersect_surr_polys = [False] * len(tl_list_copy)
for i in range(len(tl_list_copy)):
tl1, tl1_surr_poly, tl1_bl, tl1_aid = tl_list_copy[i]
for j in range(i + 1, len(tl_list_copy)):
tl2, tl2_surr_poly, tl2_bl, tl2_aid = tl_list_copy[j]
def baseline_intersection_loop(bl1, bl2):
intersect = bl1.bounds.intersection(bl2.bounds)
while intersect.width >= 0 and intersect.height >= 0:
# TODO: Check if this works (bounding boxes intersect in a horizontal way)
if intersect.height == bl1.bounds.height or intersect.height == bl2.bounds.height:
width_shift = 1
# bl1 lies right of bl2
if bl1.bounds.x + bl1.bounds.width > bl2.bounds.x + bl2.bounds.width:
bl1.bounds.width -= width_shift
bl1.bounds.x += width_shift
bl2.bounds.width -= width_shift
# bl1 lies left of bl2
else:
bl1.bounds.width -= width_shift
bl2.bounds.x += width_shift
bl2.bounds.width -= width_shift
elif bl1.bounds.y + bl1.bounds.height > bl2.bounds.y + bl2.bounds.height:
height_shift = max(1, int(0.05 * bl1.bounds.height))
bl1.bounds.height -= height_shift
bl1.bounds.y += height_shift
elif bl2.bounds.y + bl2.bounds.height > bl1.bounds.y + bl1.bounds.height:
height_shift = max(1, int(0.05 * bl2.bounds.height))
bl2.bounds.height -= height_shift
bl2.bounds.y += height_shift
intersect = bl1.bounds.intersection(bl2.bounds)
return bl1
if tl1_surr_poly is not None and not has_intersect_surr_polys[i]:
if tl2_surr_poly is not None and not has_intersect_surr_polys[j]:
intersection = tl1_surr_poly.intersection(tl2_surr_poly)
has_intersect_surr_polys[
j] = True if intersection.width >= 0 and intersection.height >= 0 else False
else:
intersection = tl1_surr_poly.intersection(tl2_bl.bounds)
if not (intersection.width >= 0 and intersection.height >= 0 and tl1_aid != tl2_aid):
if j == len(tl_list_copy) - 1:
tl_surr_poly_final.append((tl1, tl1_surr_poly, tl1_aid))
continue
has_intersect_surr_polys[i] = True
else:
if tl2_surr_poly is not None:
intersection = tl1_bl.bounds.intersection(tl2_surr_poly)
has_intersect_surr_polys[
j] = True if intersection.width >= 0 and intersection.height >= 0 else False
else:
intersection = tl1_bl.bounds.intersection(tl2_bl.bounds)
if intersection.width >= 0 and intersection.height >= 0 and tl1_aid != tl2_aid:
bl = baseline_intersection_loop(tl1_bl, tl2_bl)
if j == len(tl_list_copy) - 1:
tl_surr_poly_final.append((tl1, bl.bounds, tl1_aid))
elif j == len(tl_list_copy) - 1:
tl_surr_poly_final.append((tl1, tl1_bl.bounds, tl1_aid))
if len(has_intersect_surr_polys) > 0:
if has_intersect_surr_polys[-1] or tl_list_copy[-1][1] is None:
tl_surr_poly_final.append((tl_list_copy[-1][0], tl_list_copy[-1][2].bounds, tl_list_copy[-1][3]))
else:
tl_surr_poly_final.append((tl_list_copy[-1][0], tl_list_copy[-1][1], tl_list_copy[-1][3]))
return tl_surr_poly_final