def test_happy(self) -> None:
"""Tests the happy path."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
filters = relation.get_street_ranges()
expected_filters = {
"Budaörsi út":
ranges.Ranges([make_range(137, 165)]),
"Csiki-hegyek utca":
ranges.Ranges([make_range(1, 15),
make_range(2, 26)]),
'Hamzsabégi út':
ranges.Ranges([ranges.Range(start=1, end=12, interpolation="all")])
}
self.assertEqual(filters, expected_filters)
expected_streets = {
'OSM Name 1': 'Ref Name 1',
'OSM Name 2': 'Ref Name 2',
'Misspelled OSM Name 1': 'OSM Name 1',
}
relations = areas.Relations(test_context.make_test_context())
self.assertEqual(
relations.get_relation("gazdagret").get_config().get_refstreets(),
expected_streets)
street_blacklist = relations.get_relation(
"gazdagret").get_config().get_street_filters()
self.assertEqual(street_blacklist, ['Only In Ref Nonsense utca'])
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
relations = areas.Relations(ctx)
# Expect an alias -> canonicalname map.
expected = {"budapest_22": "budafok"}
self.assertEqual(relations.get_aliases(), expected)
def test_invalid_simplify(self) -> None:
"""Tests how 'invalid' interacts with housenumber-letters: true or false."""
relations = areas.Relations(test_context.make_test_context())
relation_name = "gh385"
relation = relations.get_relation(relation_name)
# Default case: housenumber-letters=false.
filters: Dict[str, Any] = {"Kővirág sor": {"invalid": ["37b"]}}
relation.get_config().set_filters(filters)
ongoing_streets, _done_streets = relation.get_missing_housenumbers()
# Note how 37b from invalid is simplified to 37; and how 37/B from ref is simplified to
# 37 as well, so we find the match.
self.assertFalse(len(ongoing_streets))
# Opt-in case: housenumber-letters=true.
relation.get_config().set_housenumber_letters(True)
filters = {"Kővirág sor": {"invalid": ["37b"]}}
relation.get_config().set_filters(filters)
ongoing_streets, _done_streets = relation.get_missing_housenumbers()
# In this case 37b from invalid matches 37/B from ref.
self.assertFalse(len(ongoing_streets))
# Make sure out-of-range invalid elements are just ignored and no exception is raised.
relation.get_config().set_housenumber_letters(False)
filters = {
"Kővirág sor": {
"invalid": ["5"],
"ranges": [{
"start": "1",
"end": "3"
}],
}
}
relation.get_config().set_filters(filters)
relation.get_missing_housenumbers()
def main() -> None:
"""Commandline interface to this module."""
util.set_locale()
workdir = util.Config.get_workdir()
relations = areas.Relations(workdir)
logpath = os.path.join(workdir, "cron.log")
logging.basicConfig(filename=logpath,
level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
start = time.time()
relations.activate_all(util.Config.get_cron_update_inactive())
try:
our_main(relations)
# pylint: disable=broad-except
except Exception:
logging.error("main: unhandled exception: %s", traceback.format_exc())
delta = time.time() - start
logging.info("main: finished in %s",
str(datetime.timedelta(seconds=delta)))
logging.getLogger().removeHandler(handler)
logging.shutdown()
def test_happy(self) -> None:
"""Tests the happy path."""
relation_name = "ujbuda"
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation(relation_name)
ret = relation.get_config().should_check_missing_streets()
self.assertEqual(ret, "only")
def main() -> None:
"""Commandline interface to this module."""
config = configparser.ConfigParser()
config_path = util.get_abspath("wsgi.ini")
config.read(config_path)
util.set_locale(config)
workdir = util.get_workdir(config)
relations = areas.Relations(workdir)
logpath = os.path.join(workdir, "cron.log")
logging.basicConfig(filename=logpath,
level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
start = time.time()
# Query inactive relations once a month.
relations.activate_all(time.localtime(start).tm_mday == 1)
try:
our_main(relations, config)
# pylint: disable=broad-except
except Exception:
logging.error("main: unhandled exception: %s", traceback.format_exc())
delta = time.time() - start
logging.info("main: finished in %s", str(datetime.timedelta(seconds=delta)))
logging.getLogger().removeHandler(handler)
logging.shutdown()
def test_happy(self) -> None:
"""Tests the happy path."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("test")
actual = [i.get_osm_name() for i in relation.get_osm_streets()]
expected = ['B1', 'B2', 'HB1', 'HB2']
self.assertEqual(actual, expected)
def test_http_error(self) -> None:
"""Tests the case when we keep getting HTTP errors."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path=""),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
relations = areas.Relations(ctx)
for relation_name in relations.get_active_names():
if relation_name != "gazdagret":
relations.get_relation(relation_name).get_config().set_active(
False)
expected = util.get_content(relations.get_workdir(),
"streets-gazdagret.csv")
cron.update_osm_streets(ctx, relations, update=True)
# Make sure that in case we keep getting errors we give up at some stage and
# leave the last state unchanged.
actual = util.get_content(relations.get_workdir(),
"streets-gazdagret.csv")
self.assertEqual(actual, expected)
def test_street_is_node(self) -> None:
"""Tests the case when the street name is coming from a house number (node)."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gh830")
actual = relation.get_osm_streets()
self.assertEqual(len(actual), 1)
self.assertEqual(actual[0].get_osm_type(), "node")
def test_stats(self) -> None:
"""Tests the stats path."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-stats.csv"),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
file_system = test_context.TestFileSystem()
stats_value = io.BytesIO()
stats_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("workdir/stats/stats.json"): stats_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
relations = areas.Relations(ctx)
cron.our_main(ctx,
relations,
mode="stats",
update=False,
overpass=True)
self.assertTrue(stats_value.tell())
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
relations = areas.Relations(ctx)
for relation_name in relations.get_active_names():
# gellerthegy is streets=no
if relation_name not in ("gazdagret", "gellerthegy"):
relations.get_relation(relation_name).get_config().set_active(
False)
path = os.path.join(relations.get_workdir(),
"gazdagret-additional-streets.count")
expected = "1"
if os.path.exists(path):
util.get_content(path)
os.unlink(path)
cron.update_additional_streets(ctx, relations, update=True)
mtime = os.path.getmtime(path)
cron.update_additional_streets(ctx, relations, update=False)
self.assertEqual(os.path.getmtime(path), mtime)
actual = util.get_content(path).decode("utf-8")
self.assertEqual(actual, expected)
# Make sure street stat is not created for the streets=no case.
self.assertFalse(
os.path.exists(
os.path.join(relations.get_workdir(),
"gellerthegy-additional-streets.count")))
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
relations = areas.Relations(ctx)
relation = relations.get_relation("budafok")
actual = wsgi.handle_main_housenr_additional_count(ctx, relation)
self.assertIn("42 house numbers", actual.getvalue())
def main(argv: List[str], stdout: TextIO, ctx: context.Context) -> None:
"""Commandline interface."""
log_file = argv[1]
relation_create_dates: Dict[str,
datetime.date] = get_relation_create_dates(ctx)
relations = areas.Relations(ctx)
frequent_relations = get_frequent_relations(ctx, log_file)
check_top_edited_relations(ctx, frequent_relations)
# Now suggest what to change.
removals = 0
additions = 0
for relation_name in relations.get_names():
relation = relations.get_relation(relation_name)
actual = relation.get_config().is_active()
expected = relation_name in frequent_relations and not is_complete_relation(
relations, relation_name)
if actual != expected:
if actual:
if not is_relation_recently_added(ctx, relation_create_dates,
relation_name):
stdout.write(
"data/relation-{}.yaml: set inactive: true\n".format(
relation_name))
removals += 1
else:
stdout.write(
"data/relation-{}.yaml: set inactive: false\n".format(
relation_name))
additions += 1
stdout.write("Suggested {} removals and {} additions.\n".format(
removals, additions))
def main() -> None:
"""Commandline interface."""
log_file = sys.argv[1]
relation_create_dates: Dict[str,
datetime.date] = get_relation_create_dates()
relations = areas.Relations(config.Config.get_workdir())
frequent_relations = get_frequent_relations(relations, log_file)
check_top_edited_relations(frequent_relations, config.Config.get_workdir())
# Now suggest what to change.
removals = 0
additions = 0
for relation_name in relations.get_names():
relation = relations.get_relation(relation_name)
actual = relation.get_config().is_active()
expected = relation_name in frequent_relations
if actual != expected:
if actual:
if not is_relation_recently_added(relation_create_dates,
relation_name):
print("data/relation-{}.yaml: set inactive: true".format(
relation_name))
removals += 1
else:
print("data/relation-{}.yaml: set inactive: false".format(
relation_name))
additions += 1
print("Suggested {} removals and {} additions.".format(
removals, additions))
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
relations = areas.Relations(ctx)
self.assertEqual(relations.refcounty_get_refsettlement_ids("01"),
["011", "012"])
self.assertEqual(relations.refcounty_get_refsettlement_ids("99"), [])
def test_happy(self) -> None:
"""Tests the happy path."""
relations = areas.Relations(test_context.make_test_context())
relation_name = "gazdagret"
relation = relations.get_relation(relation_name)
ret = relation.get_osm_housenumbers_query()
self.assertEqual(ret, 'housenr aaa 2713748 bbb 3602713748 ccc\n')
def test_nosuchrelation(self) -> None:
"""Tests a relation without a filter file."""
relation_name = "nosuchrelation"
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation(relation_name)
ret = relation.get_config().should_check_missing_streets()
self.assertEqual(ret, "yes")
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-streets-gazdagret.csv"),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
relations = areas.Relations(ctx)
for relation_name in relations.get_active_names():
if relation_name != "gazdagret":
relations.get_relation(relation_name).get_config().set_active(
False)
expected = util.get_content(relations.get_workdir(),
"streets-gazdagret.csv")
path = os.path.join(relations.get_workdir(), "streets-gazdagret.csv")
os.unlink(path)
cron.update_osm_streets(ctx, relations, update=True)
mtime = os.path.getmtime(path)
cron.update_osm_streets(ctx, relations, update=False)
self.assertEqual(os.path.getmtime(path), mtime)
actual = util.get_content(relations.get_workdir(),
"streets-gazdagret.csv")
self.assertEqual(actual, expected)
def our_application(
environ: Dict[str, Any],
start_response: 'StartResponse'
) -> Iterable[bytes]:
"""Dispatches the request based on its URI."""
util.set_locale()
language = util.setup_localization(environ)
relations = areas.Relations(config.Config.get_workdir())
request_uri = webframe.get_request_uri(environ, relations)
_, _, ext = request_uri.partition('.')
if ext in ("txt", "chkl"):
return our_application_txt(environ, start_response, relations, request_uri)
prefix = config.Config.get_uri_prefix()
found = request_uri == "/" or request_uri.startswith(prefix)
if not found:
doc = webframe.handle_404()
return webframe.send_response(environ,
start_response,
webframe.ResponseProperties("text/html", "404 Not Found"),
doc.getvalue().encode("utf-8"),
[])
if request_uri.startswith(prefix + "/static/") or request_uri.endswith("favicon.ico"):
output, content_type, extra_headers = webframe.handle_static(request_uri)
return webframe.send_response(environ,
start_response,
webframe.ResponseProperties(content_type, "200 OK"),
output,
extra_headers)
if ext == "json":
return wsgi_json.our_application_json(environ, start_response, relations, request_uri)
doc = yattag.doc.Doc()
util.write_html_header(doc)
with doc.tag("html", lang=language):
write_html_head(doc, get_html_title(request_uri))
with doc.tag("body"):
no_such_relation = webframe.check_existing_relation(relations, request_uri)
handler = get_handler(request_uri)
if no_such_relation.getvalue():
doc.asis(no_such_relation.getvalue())
elif handler:
doc.asis(handler(relations, request_uri).getvalue())
elif request_uri.startswith(prefix + "/webhooks/github"):
doc.asis(handle_github_webhook(environ).getvalue())
else:
doc.asis(handle_main(request_uri, relations).getvalue())
return webframe.send_response(environ,
start_response,
webframe.ResponseProperties("text/html", "200 OK"),
doc.getvalue().encode("utf-8"),
[])
def test_no_house_number(self) -> None:
"""Tests the case when we have streets, but no house numbers."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("ujbuda")
actual = [i.get_osm_name() for i in relation.get_osm_streets()]
expected = ['OSM Name 1', 'Törökugrató utca', 'Tűzkő utca']
self.assertEqual(actual, expected)
def test_not_a_number(self) -> None:
"""Tests the case when the house number is not a number."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
normalizers = relation.get_street_ranges()
house_numbers = areas.normalize(relation, "x", "Budaörsi út",
normalizers)
self.assertEqual(house_numbers, [])
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
relations = areas.Relations(ctx)
self.assertEqual(relations.refsettlement_get_name("01", "011"),
"Újbuda")
self.assertEqual(relations.refsettlement_get_name("99", ""), "")
self.assertEqual(relations.refsettlement_get_name("01", "99"), "")
def test_separator_semicolon(self) -> None:
"""Tests the case when ';' is a separator."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
normalizers = relation.get_street_ranges()
house_numbers = areas.normalize(relation, "1;2", "Budaörs út",
normalizers)
self.assertEqual([i.get_number() for i in house_numbers], ["1", "2"])
def test_addr_place(self) -> None:
"""Tests the case when addr:place is used instead of addr:street."""
relation_name = "gh964"
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation(relation_name)
street_name = "Tolvajos tanya"
house_numbers = relation.get_osm_housenumbers(street_name)
self.assertEqual([i.get_number() for i in house_numbers], ["52"])
def test_happy(self) -> None:
"""Tests the happy path."""
relation_name = "gazdagret"
street_name = "Törökugrató utca"
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation(relation_name)
house_numbers = relation.get_osm_housenumbers(street_name)
self.assertEqual([i.get_number() for i in house_numbers], ["1", "2"])
def test_separator_interval_parity(self) -> None:
"""Tests the 5-8 case: means just 5 and 8 as the parity doesn't match."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
normalizers = relation.get_street_ranges()
house_numbers = areas.normalize(relation, "5-8", "Budaörs út",
normalizers)
self.assertEqual([i.get_number() for i in house_numbers], ["5", "8"])
def test_empty(self) -> None:
"""Tests the default value."""
relation_name = "empty"
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation(relation_name)
self.assertEqual(relation.get_name(), "empty")
ret = relation.get_config().should_check_missing_streets()
self.assertEqual(ret, "yes")
def test_nofilter(self) -> None:
"""Tests the case when there is no filter for this street."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
normalizers = relation.get_street_ranges()
house_numbers = areas.normalize(relation, "1", "Budaörs út",
normalizers)
self.assertEqual([i.get_number() for i in house_numbers], ["1"])
def test_view_result_chkl_no_osm_streets_hn(self) -> None:
"""Tests the chkl output, no osm streets/hn case."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
hide_path = relation.get_files().get_osm_streets_path()
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([hide_path])
self.ctx.set_file_system(file_system)
result = self.get_txt_for_path("/missing-housenumbers/gazdagret/view-result.chkl")
self.assertEqual(result, "No existing streets")
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
hide_path = relation.get_files().get_osm_housenumbers_path()
file_system.set_hide_paths([hide_path])
result = self.get_txt_for_path("/missing-housenumbers/gazdagret/view-result.chkl")
self.assertEqual(result, "No existing house numbers")
def test_happy(self) -> None:
"""Tests the happy path."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
normalizers = relation.get_street_ranges()
house_numbers = areas.normalize(relation, "139", "Budaörsi út",
normalizers)
self.assertEqual([i.get_number() for i in house_numbers], ["139"])