def write_missing_housenumbers(
self) -> Tuple[int, int, int, str, List[List[yattag.doc.Doc]]]:
"""
Calculate a write stat for the house number coverage of a relation.
Returns a tuple of: todo street count, todo count, done count, percent and table.
"""
ongoing_streets, done_streets = self.get_missing_housenumbers()
todo_count = 0
table = []
table.append([
util.html_escape(_("Street name")),
util.html_escape(_("Missing count")),
util.html_escape(_("House numbers"))
])
rows = []
for result in ongoing_streets:
# street, only_in_ref
row = []
row.append(result[0].to_html())
number_ranges = util.get_housenumber_ranges(result[1])
row.append(util.html_escape(str(len(number_ranges))))
doc = yattag.doc.Doc()
if not self.get_config().get_street_is_even_odd(
result[0].get_osm_name()):
for index, item in enumerate(
sorted(number_ranges,
key=util.split_house_number_range)):
if index:
doc.text(", ")
doc.asis(util.color_house_number(item).getvalue())
else:
util.format_even_odd(number_ranges, doc)
row.append(doc)
todo_count += len(number_ranges)
rows.append(row)
# It's possible that get_housenumber_ranges() reduces the # of house numbers, e.g. 2, 4 and
# 6 may be turned into 2-6, which is just 1 item. Sort by the 2nd col, which is the new
# number of items.
table += sorted(rows,
reverse=True,
key=lambda cells: int(cells[1].getvalue()))
done_count = 0
for result in done_streets:
number_ranges = util.get_housenumber_ranges(result[1])
done_count += len(number_ranges)
if done_count > 0 or todo_count > 0:
percent = "%.2f" % (done_count / (done_count + todo_count) * 100)
else:
percent = "100.00"
# Write the bottom line to a file, so the index page show it fast.
with self.get_files().get_housenumbers_percent_stream("w") as stream:
stream.write(percent)
return len(ongoing_streets), todo_count, done_count, percent, table
def write_missing_housenumbers(
self) -> Tuple[int, int, int, str, List[List[yattag.doc.Doc]]]:
"""
Calculate a write stat for the house number coverage of a relation.
Returns a tuple of: todo street count, todo count, done count, percent and table.
"""
ongoing_streets, done_streets = self.get_missing_housenumbers()
todo_count = 0
table = []
table.append([
util.html_escape(_("Street name")),
util.html_escape(_("Missing count")),
util.html_escape(_("House numbers"))
])
for result in ongoing_streets:
# street_name, only_in_ref
row = []
row.append(util.html_escape(result[0]))
number_ranges = util.get_housenumber_ranges(result[1])
row.append(util.html_escape(str(len(number_ranges))))
number_range_strings = [i.get_number() for i in number_ranges]
doc = yattag.doc.Doc()
if not self.get_config().get_street_is_even_odd(result[0]):
for index, item in enumerate(
sorted(number_range_strings,
key=util.split_house_number)):
if index:
doc.text(", ")
doc.asis(util.color_house_number(item).getvalue())
else:
util.format_even_odd(number_ranges, doc)
row.append(doc)
todo_count += len(number_ranges)
table.append(row)
done_count = 0
for result in done_streets:
number_ranges = util.get_housenumber_ranges(result[1])
done_count += len(number_ranges)
if done_count > 0 or todo_count > 0:
percent = "%.2f" % (done_count / (done_count + todo_count) * 100)
else:
percent = "100.00"
# Write the bottom line to a file, so the index page show it fast.
with self.get_files().get_housenumbers_percent_stream("w") as stream:
stream.write(percent)
return len(ongoing_streets), todo_count, done_count, percent, table
def missing_housenumbers_view_txt(relations: areas.Relations,
request_uri: str) -> str:
"""Expected request_uri: e.g. /osm/missing-housenumbers/ormezo/view-result.txt."""
tokens = request_uri.split("/")
relation_name = tokens[-2]
relation = relations.get_relation(relation_name)
relation.get_config().set_letter_suffix_style(util.LetterSuffixStyle.LOWER)
output = ""
if not os.path.exists(relation.get_files().get_osm_streets_path()):
output += _("No existing streets")
elif not os.path.exists(relation.get_files().get_osm_housenumbers_path()):
output += _("No existing house numbers")
elif not os.path.exists(relation.get_files().get_ref_housenumbers_path()):
output += _("No reference house numbers")
else:
ongoing_streets, _ignore = relation.get_missing_housenumbers()
table = []
for result in ongoing_streets:
range_list = util.get_housenumber_ranges(result[1])
range_strings = [i.get_number() for i in range_list]
# Street name, only_in_reference items.
if not relation.get_config().get_street_is_even_odd(result[0]):
result_sorted = sorted(range_strings,
key=util.split_house_number)
row = result[0] + "\t[" + ", ".join(result_sorted) + "]"
else:
elements = util.format_even_odd(range_list, doc=None)
row = result[0] + "\t[" + "], [".join(elements) + "]"
table.append(row)
table.sort(key=locale.strxfrm)
output += "\n".join(table)
return output
def write_missing_housenumbers(
self) -> Tuple[int, int, int, str, List[List[yattag.doc.Doc]]]:
"""
Calculate a write stat for the house number coverage of a relation.
Returns a tuple of: todo street count, todo count, done count, percent and table.
"""
ongoing_streets, done_streets = self.get_missing_housenumbers()
table, todo_count = self.numbered_streets_to_table(ongoing_streets)
done_count = 0
for result in done_streets:
number_ranges = util.get_housenumber_ranges(result[1])
done_count += len(number_ranges)
if done_count > 0 or todo_count > 0:
percent = "%.2f" % (done_count / (done_count + todo_count) * 100)
else:
percent = "100.00"
# Write the bottom line to a file, so the index page show it fast.
with self.get_files().get_housenumbers_percent_stream(
self.__ctx, "wb") as stream:
stream.write(util.to_bytes(percent))
return len(ongoing_streets), todo_count, done_count, percent, table
def get_missing_housenumbers_txt(ctx: context.Context,
relation: areas.Relation) -> str:
"""Gets the cached plain text of the missing housenumbers for a relation."""
output = ""
if is_missing_housenumbers_txt_cached(ctx, relation):
with relation.get_files().get_housenumbers_txtcache_stream(
"rb") as stream:
output = util.from_bytes(stream.read())
return output
ongoing_streets, _ignore = relation.get_missing_housenumbers()
table = []
for result in ongoing_streets:
range_list = util.get_housenumber_ranges(result[1])
range_strings = [i.get_number() for i in range_list]
# Street name, only_in_reference items.
if not relation.get_config().get_street_is_even_odd(
result[0].get_osm_name()):
result_sorted = sorted(range_strings, key=util.split_house_number)
row = result[0].get_osm_name() + "\t[" + ", ".join(
result_sorted) + "]"
else:
elements = util.format_even_odd(range_list, doc=None)
row = result[0].get_osm_name() + "\t[" + "], [".join(
elements) + "]"
table.append(row)
table.sort(key=util.get_lexical_sort_key())
output += "\n".join(table)
with relation.get_files().get_housenumbers_txtcache_stream("wb") as stream:
stream.write(util.to_bytes(output))
return output
def test_happy(self) -> None:
"""Tests the happy path."""
house_numbers = [
util.HouseNumber("25", "25"),
util.HouseNumber("27", "27-37"),
util.HouseNumber("29", "27-37"),
util.HouseNumber("31", "27-37"),
util.HouseNumber("33", "27-37"),
util.HouseNumber("35", "27-37"),
util.HouseNumber("37", "27-37"),
util.HouseNumber("31*", "31*"),
]
ranges = util.get_housenumber_ranges(house_numbers)
self.assertEqual(ranges, ["25", "27-37", "31*"])
def test_letter_suffix(self) -> None:
"""Tests that 7/A is detected when 7/B is already mapped."""
with unittest.mock.patch('util.get_abspath', get_abspath):
relations = get_relations()
relation_name = "gh267"
relation = relations.get_relation(relation_name)
# Opt-in, this is not the default behavior.
relation.get_config().set_housenumber_letters(True)
ongoing_streets, _done_streets = relation.get_missing_housenumbers()
ongoing_street = ongoing_streets[0]
housenumber_ranges = util.get_housenumber_ranges(ongoing_street[1])
housenumber_ranges = sorted(housenumber_ranges, key=util.split_house_number)
expected = ['1', '3', '5', '7', '7/A', '7/B', '7/C', '9', '11', '13', '13-15']
self.assertEqual(housenumber_ranges, expected)
def missing_housenumbers_view_chkl(ctx: context.Context,
relations: areas.Relations,
request_uri: str) -> Tuple[str, str]:
"""Expected request_uri: e.g. /osm/missing-housenumbers/ormezo/view-result.chkl."""
tokens = request_uri.split("/")
relation_name = tokens[-2]
relation = relations.get_relation(relation_name)
relation.get_config().set_letter_suffix_style(util.LetterSuffixStyle.LOWER)
output = ""
if not ctx.get_file_system().path_exists(
relation.get_files().get_osm_streets_path()):
output += tr("No existing streets")
elif not ctx.get_file_system().path_exists(
relation.get_files().get_osm_housenumbers_path()):
output += tr("No existing house numbers")
elif not ctx.get_file_system().path_exists(
relation.get_files().get_ref_housenumbers_path()):
output += tr("No reference house numbers")
else:
ongoing_streets, _ignore = relation.get_missing_housenumbers()
table = []
for result in ongoing_streets:
range_list = util.get_housenumber_ranges(result[1])
# Street name, only_in_reference items.
row = "[ ] "
if not relation.get_config().get_street_is_even_odd(
result[0].get_osm_name()):
result_sorted = sorted([i.get_number() for i in range_list],
key=util.split_house_number)
row += result[0].get_osm_name() + " [" + ", ".join(
result_sorted) + "]"
table.append(row)
else:
elements = util.format_even_odd(range_list, doc=None)
if len(elements) > 1 and len(
range_list) > get_chkl_split_limit():
for element in elements:
row = "[ ] " + result[0].get_osm_name(
) + " [" + element + "]"
table.append(row)
else:
row += result[0].get_osm_name() + " [" + "], [".join(
elements) + "]"
table.append(row)
table.sort(key=util.get_lexical_sort_key())
output += "\n".join(table)
return output, relation_name
def test_letter_suffix_normalize_semicolon(self) -> None:
"""Tests that 'a' is not stripped from '1;3a'."""
relations = get_relations()
relation_name = "gh303"
relation = relations.get_relation(relation_name)
# Opt-in, this is not the default behavior.
relation.get_config().set_housenumber_letters(True)
ongoing_streets, _done_streets = relation.get_missing_housenumbers()
ongoing_street = ongoing_streets[0]
housenumber_ranges = util.get_housenumber_ranges(ongoing_street[1])
housenumber_range_names = [i.get_number() for i in housenumber_ranges]
housenumber_range_names = sorted(housenumber_range_names, key=util.split_house_number)
# Note how 43/B and 43/C is not here.
expected = ['43/A', '43/D']
self.assertEqual(housenumber_range_names, expected)
def test_letter_suffix_normalize(self) -> None:
"""Tests that '42 A' vs '42/A' is recognized as a match."""
relations = get_relations()
relation_name = "gh286"
relation = relations.get_relation(relation_name)
# Opt-in, this is not the default behavior.
relation.get_config().set_housenumber_letters(True)
ongoing_streets, _done_streets = relation.get_missing_housenumbers()
ongoing_street = ongoing_streets[0]
housenumber_ranges = util.get_housenumber_ranges(ongoing_street[1])
housenumber_range_names = [i.get_number() for i in housenumber_ranges]
housenumber_range_names = sorted(housenumber_range_names, key=util.split_house_number)
# Note how 10/B is not in this list.
expected = ['10/A']
self.assertEqual(housenumber_range_names, expected)
def test_letter_suffix(self) -> None:
"""Tests that 7/A is detected when 7/B is already mapped."""
relations = get_relations()
relation_name = "gh267"
relation = relations.get_relation(relation_name)
# Opt-in, this is not the default behavior.
relation.get_config().set_housenumber_letters(True)
ongoing_streets, _done_streets = relation.get_missing_housenumbers()
ongoing_street = ongoing_streets[0]
housenumber_ranges = util.get_housenumber_ranges(ongoing_street[1])
housenumber_range_names = [i.get_number() for i in housenumber_ranges]
housenumber_range_names = sorted(housenumber_range_names, key=util.split_house_number)
# Make sure that 1/1 shows up in the output: it's not the same as '1' or '11'.
expected = ['1', '1/1', '1/2', '3', '5', '7', '7/A', '7/B', '7/C', '9', '11', '13', '13-15']
self.assertEqual(housenumber_range_names, expected)
def main() -> None:
"""Commandline interface."""
workdir = util.Config.get_workdir()
relation_name = sys.argv[1]
relations = areas.Relations(workdir)
relation = relations.get_relation(relation_name)
ongoing_streets, _ = relation.get_missing_housenumbers()
for result in ongoing_streets:
# House number, # of only_in_reference items.
ranges = util.get_housenumber_ranges(result[1])
ranges = sorted(ranges, key=util.split_house_number)
print("%s\t%s" % (result[0], len(ranges)))
# only_in_reference items.
print(ranges)
def main(argv: List[str], stdout: TextIO, ctx: context.Context) -> None:
"""Commandline interface."""
relation_name = argv[1]
relations = areas.Relations(ctx)
relation = relations.get_relation(relation_name)
ongoing_streets, _ = relation.get_missing_housenumbers()
for result in ongoing_streets:
# House number, # of only_in_reference items.
range_list = util.get_housenumber_ranges(result[1])
range_strings = [i.get_number() for i in range_list]
range_strings = sorted(range_strings, key=util.split_house_number)
stdout.write("%s\t%s\n" %
(result[0].get_osm_name(), len(range_strings)))
# only_in_reference items.
stdout.write(str(range_strings) + "\n")
def numbered_streets_to_table(
self, numbered_streets: util.NumberedStreets
) -> Tuple[List[List[yattag.doc.Doc]], int]:
"""Turns a list of numbered streets into a HTML table."""
todo_count = 0
table = []
table.append([
util.html_escape(tr("Street name")),
util.html_escape(tr("Missing count")),
util.html_escape(tr("House numbers"))
])
rows = []
for result in numbered_streets:
# street, only_in_ref
row = []
row.append(result[0].to_html())
number_ranges = util.get_housenumber_ranges(result[1])
row.append(util.html_escape(str(len(number_ranges))))
doc = yattag.doc.Doc()
if not self.get_config().get_street_is_even_odd(
result[0].get_osm_name()):
for index, item in enumerate(
sorted(number_ranges,
key=util.split_house_number_range)):
if index:
doc.text(", ")
doc.asis(util.color_house_number(item).getvalue())
else:
util.format_even_odd(number_ranges, doc)
row.append(doc)
todo_count += len(number_ranges)
rows.append(row)
# It's possible that get_housenumber_ranges() reduces the # of house numbers, e.g. 2, 4 and
# 6 may be turned into 2-6, which is just 1 item. Sort by the 2nd col, which is the new
# number of items.
table += sorted(rows,
reverse=True,
key=lambda cells: int(cells[1].getvalue()))
return table, todo_count
def main() -> None:
"""Commandline interface."""
config = configparser.ConfigParser()
config_path = util.get_abspath("wsgi.ini")
config.read(config_path)
workdir = util.get_abspath(config.get('wsgi', 'workdir').strip())
relation_name = sys.argv[1]
relations = areas.Relations(workdir)
relation = relations.get_relation(relation_name)
ongoing_streets, _ = relation.get_missing_housenumbers()
for result in ongoing_streets:
# House number, # of only_in_reference items.
ranges = util.get_housenumber_ranges(result[1])
ranges = sorted(ranges, key=util.split_house_number)
print("%s\t%s" % (result[0], len(ranges)))
# only_in_reference items.
print(ranges)
def test_letter_suffix_invalid(self) -> None:
"""Tests how 'invalid' interacts with normalization."""
relations = get_relations()
relation_name = "gh296"
relation = relations.get_relation(relation_name)
# Opt-in, this is not the default behavior.
relation.get_config().set_housenumber_letters(True)
# Set custom 'invalid' map.
filters = {"Rétköz utca": {"invalid": ["9", "47"]}}
relation.get_config().set_filters(filters)
ongoing_streets, _done_streets = relation.get_missing_housenumbers()
ongoing_street = ongoing_streets[0]
housenumber_ranges = util.get_housenumber_ranges(ongoing_street[1])
housenumber_range_names = [i.get_number() for i in housenumber_ranges]
housenumber_range_names = sorted(housenumber_range_names,
key=util.split_house_number)
# Notice how '9 A 1' is missing here: it's not a simple house number, so it gets normalized
# to just '9' and the above filter silences it.
expected = ['9/A']
self.assertEqual(housenumber_range_names, expected)
