def test_edit(self, test_app, restriction_area_test_data, dbsession):
restriction_area = restriction_area_test_data['restriction_areas'][0]
roles = restriction_area_test_data['roles']
form = self.get_item(test_app, restriction_area.id).form
assert str(restriction_area.id) == self.get_first_field_named(
form, 'id').value
assert 'hidden' == self.get_first_field_named(form, 'id').attrs['type']
assert restriction_area.name == form['name'].value
expected = Polygon([(1167544.3397631699, 5748064.729594703),
(1180453.0256760044, 6074797.96820131),
(658348.6696383564, 6080273.63626964),
(664577.4194513536, 5753148.2510447875),
(1167544.3397631699, 5748064.729594703)])
assert expected.almost_equals(shape(json.loads(form['area'].value)))
self._check_roles(form, roles, restriction_area)
form['description'] = 'new_description'
form['roles'] = [roles[i].id for i in range(0, 3)]
form.submit('submit')
dbsession.expire(restriction_area)
assert restriction_area.description == 'new_description'
assert set(restriction_area.roles) == set(
[roles[i] for i in range(0, 3)])
def test_edit(self, test_app, restriction_area_test_data, dbsession):
restriction_area = restriction_area_test_data["restriction_areas"][0]
roles = restriction_area_test_data["roles"]
form = self.get_item(test_app, restriction_area.id).form
assert str(restriction_area.id) == self.get_first_field_named(
form, "id").value
assert "hidden" == self.get_first_field_named(form, "id").attrs["type"]
assert restriction_area.name == form["name"].value
expected = Polygon([
(1167544.3397631699, 5748064.729594703),
(1180453.0256760044, 6074797.96820131),
(658348.6696383564, 6080273.63626964),
(664577.4194513536, 5753148.2510447875),
(1167544.3397631699, 5748064.729594703),
])
assert expected.almost_equals(shape(json.loads(form["area"].value)),
decimal=0)
self._check_roles(form, roles, restriction_area)
form["description"] = "new_description"
form["roles"] = [roles[i].id for i in range(0, 3)]
form.submit("submit")
dbsession.expire(restriction_area)
assert restriction_area.description == "new_description"
assert set(restriction_area.roles) == set(
[roles[i] for i in range(0, 3)])
def processPastDetection (bbox, label, conf,mid):
mon_file = g.config['image_path'] + '/monitor-'+mid +'-data.pkl'
g.logger.debug ('trying to load '+mon_file)
try:
fh = open(mon_file, "rb")
saved_bs = pickle.load(fh)
saved_ls = pickle.load(fh)
saved_cs = pickle.load(fh)
except FileNotFoundError:
g.logger.debug ('No history data file found for monitor {}'.format(mid))
return bbox, label, conf
# load past detection
#g.logger.debug ('loaded past: bbox={}, labels={}'.format(saved_bs, saved_ls));
new_label = []
new_bbox = []
new_conf = []
for idx, b in enumerate(bbox):
# iterate list of detections
old_b = b
it = iter(b)
b = list(zip(it,it))
b.insert(1, (b[1][0], b[0][1]))
b.insert(3, (b[0][0], b[1][1]))
#g.logger.debug ("Past detection: {}@{}".format(saved_ls[idx],b))
#g.logger.debug ('BOBK={}'.format(b))
obj = Polygon(b)
foundMatch = False
for saved_idx, saved_b in enumerate(saved_bs):
# compare current detection element with saved list from file
if saved_ls[saved_idx] != label[idx]: continue
it = iter(saved_b)
saved_b = list(zip(it,it))
saved_b.insert(1,
(saved_b[1][0], saved_b[0][1]))
saved_b.insert(3, (saved_b[0][0], saved_b[1][1]))
saved_obj = Polygon(saved_b)
if obj.equals(saved_obj):
g.logger.debug ('past detection {}@{} exactly matches {}@{} removing'.format(saved_ls[saved_idx],saved_b, label[idx],b))
foundMatch = True
break
if obj.almost_equals(saved_obj):
g.logger.debug ('past detection {}@{} approximately matches {}@{} removing'.format(saved_ls[saved_idx],saved_b, label[idx],b))
foundMatch = True
break
if not foundMatch:
new_bbox.append(old_b)
new_label.append(label[idx])
new_conf.append(conf[idx])
return new_bbox, new_label, new_conf
def are_polygons_almost_equal(polygon_1: Union[str, Polygon],
polygon_2: Union[str, Polygon]):
if type(polygon_2) == str:
polygon_2 = loads(polygon_2)
if type(polygon_1 == str):
polygon_1 = loads(polygon_1)
if polygon_1.almost_equals(polygon_2):
return True
x_list, y_list = polygon_1.exterior.coords.xy
reversed_points = []
for i in range(len(x_list) - 1, -1, -1):
reversed_points.append(Point(x_list[i], y_list[i]))
reversed_polygon = Polygon([[p.x, p.y] for p in reversed_points])
return reversed_polygon.almost_equals(polygon_2)
def test_edit(self, test_app, restriction_area_test_data, dbsession):
restriction_area = restriction_area_test_data["restriction_areas"][0]
roles = restriction_area_test_data["roles"]
# Ensure restriction_area.layers is loaded with relationship "order_by"
dbsession.expire(restriction_area)
form = self.get_item(test_app, restriction_area.id).form
assert str(restriction_area.id) == self.get_first_field_named(
form, "id").value
assert "hidden" == self.get_first_field_named(form, "id").attrs["type"]
assert restriction_area.name == form["name"].value
expected = Polygon([
(1167544.3397631699, 5748064.729594703),
(1180453.0256760044, 6074797.96820131),
(658348.6696383564, 6080273.63626964),
(664577.4194513536, 5753148.2510447875),
(1167544.3397631699, 5748064.729594703),
])
assert expected.almost_equals(shape(json.loads(form["area"].value)),
decimal=0)
self._check_roles(form, roles, restriction_area)
self.check_children(
form,
"layers",
[{
"label": layer.name,
"values": {
"id": str(layer.id)
}
}
for layer in sorted(restriction_area.layers, key=lambda l: l.name)
],
)
form["description"] = "new_description"
form["roles"] = [roles[i].id for i in range(0, 3)]
form.submit("submit")
dbsession.expire(restriction_area)
assert restriction_area.description == "new_description"
assert set(restriction_area.roles) == {roles[i] for i in range(0, 3)}
print(f"Possible points: {red(total_points)}")
print(f"Raw score: {yellow(approval_score)}")
print(f"Geoconfidence: {green(confidence / 100)}")
print(f"Perplexity penalty: {blue(coeff)}")
print(f"Percent of total points awarded: {cyan(base * coeff)}")
print(f"New score: {magenta(adjusted)}")
# continue
acc = {k: 0 for k in county_shapes.keys()}
for county, hull in county_shapes.items():
if shape:
c1 = list(sorted(list(*hull.centroid.coords), reverse=True))
c2 = list(sorted(list(*shape.centroid.coords), reverse=True))
#
distance_from_target = (
1 if hull.contains(shape) or shape.contains(hull)
or hull.almost_equals(shape) else geodesic(c1, c2).km)
else:
distance_from_target = 9999
coreference_score = 0
if audience == 'regional':
distance_penalty = 1 - max(
math.sqrt(distance_from_target * 0.2) / 2, 1)
coreference_score = adjusted if distance_from_target <= 1 else adjusted * distance_penalty
elif audience == 'national':
quality = 2 if is_high_quality_source and approval_score > 1 and row[
'docvec_v2'][6] > 4 else 0.9
coreference_score = min(8, adjusted * quality)
elif audience == 'city':
coreference_score = adjusted if county == prediction else 0
elif audience == 'state':
my_state = prediction.split(" County, ")[-1]
def test_edit(self, dbsession, test_app, roles_test_data):
role = roles_test_data['roles'][10]
form = self.get_item(test_app, role.id).form
assert 'secretary_10' == form['name'].value
expected = Polygon([(1167544.3397631699, 5748064.729594703),
(1180453.0256760044, 6074797.96820131),
(658348.6696383564, 6080273.63626964),
(664577.4194513536, 5753148.2510447875),
(1167544.3397631699, 5748064.729594703)])
assert expected.almost_equals(shape(json.loads(form['extent'].value)))
functionalities = roles_test_data['functionalities']
assert set((
functionalities['default_basemap'][0].id,
functionalities['location'][0].id,
functionalities['location'][1].id
)) == set(f.id for f in role.functionalities)
self.check_checkboxes(
form,
'functionalities',
[{
'label': '{}={}'.format(f.name, f.value),
'value': str(f.id),
'checked': f in role.functionalities
} for f in sum(
[roles_test_data['functionalities'][name] for name in ('default_basemap', 'location')],
[]
)])
ras = roles_test_data['restrictionareas']
assert set((ras[0].id, ras[1].id)) == set(ra.id for ra in role.restrictionareas)
self.check_checkboxes(
form,
'restrictionareas',
[{
'label': ra.name,
'value': str(ra.id),
'checked': ra in role.restrictionareas
} for ra in sorted(ras, key=lambda ra: ra.name)])
form['name'] = 'New name'
form['description'] = 'New description'
form['extent'] = json.dumps({
'type': 'Polygon',
'coordinates': [[
[1000000, 5800000],
[1000000, 6000000],
[700000, 6000000],
[700000, 5800000],
[1000000, 5800000]
]]
})
functionality_ids = [
roles_test_data['functionalities']['default_basemap'][1].id,
roles_test_data['functionalities']['location'][1].id,
roles_test_data['functionalities']['default_basemap'][2].id]
form['functionalities'] = [str(id) for id in functionality_ids]
ra_ids = [
roles_test_data['restrictionareas'][2].id,
roles_test_data['restrictionareas'][3].id]
form['restrictionareas'] = [str(id) for id in ra_ids]
resp = form.submit('submit')
assert str(role.id) == re.match(
r'http://localhost/roles/(.*)\?msg_col=submit_ok',
resp.location
).group(1)
dbsession.expire(role)
assert 'New name' == role.name
assert 'New description' == role.description
expected = Polygon([(719383.7988896352, 109062.8141734005),
(716689.3301397888, 245909.7643546755),
(513083.1504351135, 245400.5416369234),
(511073.1973649057, 108541.7344432737),
(719383.7988896352, 109062.8141734005)])
assert expected.almost_equals(to_shape(role.extent))
assert set(functionality_ids) == set([f.id for f in role.functionalities])
assert set(ra_ids) == set([f.id for f in role.restrictionareas])
def process(self):
"""Performs segmentation evaluation with Shapely on the workspace.
Open and deserialize PAGE input files and their respective images,
then iterate over the element hierarchy down to the region level.
Return information on the plausibility of the segmentation into
regions on the logging level.
"""
plausibilize = self.parameter['plausibilize']
for (n, input_file) in enumerate(self.input_files):
page_id = input_file.pageId or input_file.ID
LOG.info("INPUT FILE %i / %s", n, page_id)
pcgts = page_from_file(self.workspace.download_file(input_file))
metadata = pcgts.get_Metadata() # ensured by from_file()
metadata.add_MetadataItem(
MetadataItemType(
type_="processingStep",
name=self.ocrd_tool['steps'][0],
value=TOOL,
# FIXME: externalRef is invalid by pagecontent.xsd, but ocrd does not reflect this
# what we want here is `externalModel="ocrd-tool" externalId="parameters"`
Labels=[
LabelsType( #externalRef="parameters",
Label=[
LabelType(type_=name,
value=self.parameter[name])
for name in self.parameter.keys()
])
]))
page = pcgts.get_Page()
regions = page.get_TextRegion()
mark_for_deletion = set()
mark_for_merging = set()
for i in range(0, len(regions)):
for j in range(i + 1, len(regions)):
LOG.info('Comparing regions "%s" and "%s"', regions[i].id,
regions[j].id)
region_poly1 = Polygon(
polygon_from_points(regions[i].get_Coords().points))
region_poly2 = Polygon(
polygon_from_points(regions[j].get_Coords().points))
LOG.debug('Checking for equality ...')
equality = region_poly1.almost_equals(region_poly2)
if equality:
LOG.warn(
'Warning: regions %s and %s cover the same area.' %
(regions[i].id, regions[j].id))
mark_for_deletion.add(j)
LOG.debug('Checking for containment ...')
containment_r = region_poly1.contains(region_poly2)
containment_l = region_poly2.contains(region_poly1)
if containment_r:
LOG.warn('Warning: %s contains %s' %
(regions[i].id, regions[j].id))
mark_for_deletion.add(j)
if containment_l:
LOG.warn('Warning: %s contains %s' %
(regions[j].id, regions[i].id))
mark_for_deletion.add(i)
if plausibilize:
new_regions = []
for i in range(0, len(regions)):
if not i in mark_for_deletion:
new_regions.append(regions[i])
page.set_TextRegion(new_regions)
#LOG.info('Intersection %i', region_poly1.intersects(region_poly2))
#LOG.info('Containment %i', region_poly1.contains(region_poly2))
#if region_poly1.intersects(region_poly2):
# LOG.info('Area 1 %d', region_poly1.area)
# LOG.info('Area 2 %d', region_poly2.area)
# LOG.info('Area intersect %d', region_poly1.intersection(region_poly2).area)
# Use input_file's basename for the new file -
# this way the files retain the same basenames:
file_id = input_file.ID.replace(self.input_file_grp,
self.output_file_grp)
if file_id == input_file.ID:
file_id = concat_padded(self.output_file_grp, n)
self.workspace.add_file(ID=file_id,
file_grp=self.output_file_grp,
pageId=input_file.pageId,
mimetype=MIMETYPE_PAGE,
local_filename=os.path.join(
self.output_file_grp,
file_id + '.xml'),
content=to_xml(pcgts))
def test_edit(self, dbsession, test_app, roles_test_data):
role = roles_test_data["roles"][10]
# Ensure role.users is loaded with relationship "order_by"
dbsession.expire(role)
form = self.get_item(test_app, role.id).form
assert "secretary_10" == form["name"].value
expected = Polygon([
(1167544.3397631699, 5748064.729594703),
(1180453.0256760044, 6074797.96820131),
(658348.6696383564, 6080273.63626964),
(664577.4194513536, 5753148.2510447875),
(1167544.3397631699, 5748064.729594703),
])
assert expected.almost_equals(shape(json.loads(form["extent"].value)),
decimal=0)
functionalities = roles_test_data["functionalities"]
assert {
functionalities["default_theme"][0].id,
functionalities["print_template"][0].id,
functionalities["print_template"][1].id,
} == {f.id
for f in role.functionalities}
self.check_checkboxes(
form,
"functionalities",
[{
"label": f"{f.name}={f.value}",
"value": str(f.id),
"checked": f in role.functionalities,
} for f in sorted(
[
f for f in sum(functionalities.values(), [])
if f.name in ("default_theme", "print_template")
],
key=lambda f: (f.name, f.value),
)],
)
ras = roles_test_data["restrictionareas"]
assert {ras[0].id,
ras[1].id} == {ra.id
for ra in role.restrictionareas}
self.check_checkboxes(
form,
"restrictionareas",
[{
"label": ra.name,
"value": str(ra.id),
"checked": ra in role.restrictionareas
} for ra in sorted(ras, key=lambda ra: ra.name)],
)
self.check_children(
form,
"users",
[{
"label": user.username,
"values": {
"id": str(user.id)
}
} for user in sorted(role.users, key=lambda u: u.username)],
)
form["name"] = "New name"
form["description"] = "New description"
form["extent"] = json.dumps({
"type":
"Polygon",
"coordinates": [[
[1000000, 5800000],
[1000000, 6000000],
[700000, 6000000],
[700000, 5800000],
[1000000, 5800000],
]],
})
functionality_ids = [
roles_test_data["functionalities"]["default_theme"][1].id,
roles_test_data["functionalities"]["print_template"][1].id,
roles_test_data["functionalities"]["print_template"][2].id,
]
form["functionalities"] = [str(id) for id in functionality_ids]
ra_ids = [
roles_test_data["restrictionareas"][2].id,
roles_test_data["restrictionareas"][3].id
]
form["restrictionareas"] = [str(id) for id in ra_ids]
resp = form.submit("submit")
assert str(role.id) == re.match(
r"http://localhost/admin/roles/(.*)\?msg_col=submit_ok",
resp.location).group(1)
dbsession.expire(role)
assert "New name" == role.name
assert "New description" == role.description
expected = Polygon([
(719383.7988896352, 109062.8141734005),
(716689.3301397888, 245909.7643546755),
(513083.1504351135, 245400.5416369234),
(511073.1973649057, 108541.7344432737),
(719383.7988896352, 109062.8141734005),
])
assert expected.almost_equals(to_shape(role.extent), decimal=0)
assert set(functionality_ids) == {f.id for f in role.functionalities}
assert set(ra_ids) == {f.id for f in role.restrictionareas}
def test_edit(self, dbsession, test_app, roles_test_data):
role = roles_test_data['roles'][10]
form = self.get_item(test_app, role.id).form
assert 'secretary_10' == form['name'].value
expected = Polygon([(1167544.3397631699, 5748064.729594703),
(1180453.0256760044, 6074797.96820131),
(658348.6696383564, 6080273.63626964),
(664577.4194513536, 5753148.2510447875),
(1167544.3397631699, 5748064.729594703)])
assert expected.almost_equals(shape(json.loads(form['extent'].value)))
functionalities = roles_test_data['functionalities']
assert set((
functionalities['default_basemap'][0].id,
functionalities['location'][0].id,
functionalities['location'][1].id
)) == set(f.id for f in role.functionalities)
self.check_checkboxes(
form,
'functionalities',
[{
'label': '{}={}'.format(f.name, f.value),
'value': str(f.id),
'checked': f in role.functionalities
} for f in sum([roles_test_data['functionalities'][name]
for name in ('default_basemap', 'location')],
[])])
ras = roles_test_data['restrictionareas']
assert set((ras[0].id, ras[1].id)) == set(ra.id for ra in role.restrictionareas)
self.check_checkboxes(
form,
'restrictionareas',
[{
'label': ra.name,
'value': str(ra.id),
'checked': ra in role.restrictionareas
} for ra in sorted(ras, key=lambda ra: ra.name)])
form['name'] = 'New name'
form['description'] = 'New description'
form['extent'] = json.dumps({
'type': 'Polygon',
'coordinates': [[
[1000000, 5800000],
[1000000, 6000000],
[700000, 6000000],
[700000, 5800000],
[1000000, 5800000]
]]
})
functionality_ids = [
roles_test_data['functionalities']['default_basemap'][1].id,
roles_test_data['functionalities']['location'][1].id,
roles_test_data['functionalities']['default_basemap'][2].id]
form['functionalities'] = [str(id) for id in functionality_ids]
ra_ids = [
roles_test_data['restrictionareas'][2].id,
roles_test_data['restrictionareas'][3].id]
form['restrictionareas'] = [str(id) for id in ra_ids]
resp = form.submit('submit')
assert str(role.id) == re.match('http://localhost/roles/(.*)', resp.location).group(1)
dbsession.expire(role)
assert 'New name' == role.name
assert 'New description' == role.description
expected = Polygon([(719383.7988896352, 109062.8141734005),
(716689.3301397888, 245909.7643546755),
(513083.1504351135, 245400.5416369234),
(511073.1973649057, 108541.7344432737),
(719383.7988896352, 109062.8141734005)])
assert expected.almost_equals(to_shape(role.extent))
assert set(functionality_ids) == set([f.id for f in role.functionalities])
assert set(ra_ids) == set([f.id for f in role.restrictionareas])
def process(self):
"""Performs segmentation evaluation with Shapely on the workspace.
Open and deserialize PAGE input files and their respective images,
then iterate over the element hierarchy down to the region level.
Return information on the plausibility of the segmentation into
regions on the logging level.
"""
sanitize = self.parameter['sanitize']
plausibilize = self.parameter['plausibilize']
for (n, input_file) in enumerate(self.input_files):
page_id = input_file.pageId or input_file.ID
LOG.info("INPUT FILE %i / %s", n, page_id)
pcgts = page_from_file(self.workspace.download_file(input_file))
page = pcgts.get_Page()
metadata = pcgts.get_Metadata() # ensured by from_file()
metadata.add_MetadataItem(
MetadataItemType(type_="processingStep",
name=self.ocrd_tool['steps'][0],
value=TOOL,
Labels=[LabelsType(
externalModel="ocrd-tool",
externalId="parameters",
Label=[LabelType(type_=name,
value=self.parameter[name])
for name in self.parameter.keys()])]))
#
# validate segmentation (warn of children extending beyond their parents)
#
self.validate_coords(page, page_id)
#
# sanitize region segmentation (shrink to hull of lines)
#
if sanitize:
self.sanitize_page(page, page_id)
#
# plausibilize region segmentation (remove redundant regions)
#
mark_for_deletion = list()
mark_for_merging = list()
regions = page.get_TextRegion()
for i in range(0, len(regions)):
for j in range(i+1, len(regions)):
region1 = regions[i]
region2 = regions[j]
LOG.debug('Comparing regions "%s" and "%s"', region1.id, region2.id)
region1_poly = Polygon(polygon_from_points(region1.get_Coords().points))
region2_poly = Polygon(polygon_from_points(region2.get_Coords().points))
equality = region1_poly.almost_equals(region2_poly)
if equality:
LOG.warning('Page "%s" regions "%s" and "%s" cover the same area.',
page_id, region1.id, region2.id)
mark_for_deletion.append(region2)
if region1_poly.contains(region2_poly):
LOG.warning('Page "%s" region "%s" contains "%s"',
page_id, region1.id, region2.id)
mark_for_deletion.append(region2)
elif region2_poly.contains(region1_poly):
LOG.warning('Page "%s" region "%s" contains "%s"',
page_id, region2.id, region1.id)
mark_for_deletion.append(region1)
#LOG.info('Intersection %i', region1_poly.intersects(region2_poly))
#LOG.info('Containment %i', region1_poly1.contains(region2_poly))
#if region1_poly.intersects(region2_poly):
# LOG.info('Area 1 %d', region1_poly.area)
# LOG.info('Area 2 %d', region2_poly.area)
# LOG.info('Area intersect %d', region1_poly.intersection(region2_poly).area)
if plausibilize:
# the reading order does not have to include all regions
# but it may include all types of regions!
ro = page.get_ReadingOrder()
_plausibilize_group(regions, ro.get_OrderedGroup() or ro.get_UnorderedGroup(),
mark_for_deletion)
# Use input_file's basename for the new file -
# this way the files retain the same basenames:
file_id = input_file.ID.replace(self.input_file_grp, self.output_file_grp)
if file_id == input_file.ID:
file_id = concat_padded(self.output_file_grp, n)
self.workspace.add_file(
ID=file_id,
file_grp=self.output_file_grp,
pageId=input_file.pageId,
mimetype=MIMETYPE_PAGE,
local_filename=os.path.join(self.output_file_grp,
file_id + '.xml'),
content=to_xml(pcgts))
