def test_stats(self) -> None:
"""Tests the stats path."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-stats.csv"),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
file_system = test_context.TestFileSystem()
stats_value = io.BytesIO()
stats_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("workdir/stats/stats.json"): stats_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
relations = areas.Relations(ctx)
cron.our_main(ctx,
relations,
mode="stats",
update=False,
overpass=True)
self.assertTrue(stats_value.tell())
def test_happy(self) -> None:
"""Tests tha happy path."""
ctx = test_context.make_test_context()
file_system = test_context.TestFileSystem()
today_csv = util.to_bytes(
"""addr:postcode addr:city addr:street addr:housenumber @user
7677 Orfű Dollár utca 1 mgpx
""")
today_csv_value = io.BytesIO(today_csv)
today_csv_value.__setattr__("close", lambda: None)
today_topusers_value = io.BytesIO()
today_topusers_value.__setattr__("close", lambda: None)
today_usercount_value = io.BytesIO()
today_usercount_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("workdir/stats/2020-05-10.csv"):
today_csv_value,
ctx.get_abspath("workdir/stats/2020-05-10.topusers"):
today_topusers_value,
ctx.get_abspath("workdir/stats/2020-05-10.usercount"):
today_usercount_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
cron.update_stats_topusers(ctx, "2020-05-10")
self.assertTrue(today_topusers_value.tell())
self.assertTrue(today_usercount_value.tell())
def test_view_result_txt_no_ref_streets(self) -> None:
"""Tests the txt output, no ref streets case."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
hide_path = relation.get_files().get_ref_streets_path()
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([hide_path])
self.ctx.set_file_system(file_system)
result = self.get_txt_for_path("/missing-streets/gazdagret/view-result.txt")
self.assertEqual(result, "No reference streets")
def test_view_result_txt_no_osm_housenumbers(self) -> None:
"""Tests the txt output, no osm housenumbers case."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
hide_path = relation.get_files().get_osm_housenumbers_path()
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([hide_path])
self.ctx.set_file_system(file_system)
result = self.get_txt_for_path("/missing-housenumbers/gazdagret/view-result.txt")
self.assertEqual(result, "No existing house numbers")
def test_new_missing(self) -> None:
"""Tests the case when the new path is missing."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
file_system = test_context.TestFileSystem()
src_root = ctx.get_abspath("workdir/stats")
file_system.set_hide_paths(
[os.path.join(src_root, "2020-05-10.citycount")])
ctx.set_file_system(file_system)
ret = stats.get_topcities(ctx, src_root)
self.assertEqual(ret, [])
def test_no_osm_sreets(self) -> None:
"""Tests error handling when osm street list is missing for a relation."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
hide_path = relation.get_files().get_osm_streets_path()
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([hide_path])
self.ctx.set_file_system(file_system)
root = self.get_dom_for_path("/housenumber-stats/hungary/invalid-relations")
results = root.findall("body")
self.assertNotEqual(results, [])
def test_no_ref_streets_well_formed(self) -> None:
"""Tests if the output is well-formed, no ref streets case."""
relations = areas.Relations(test_context.make_test_context())
relation = relations.get_relation("gazdagret")
hide_path = relation.get_files().get_ref_streets_path()
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([hide_path])
self.ctx.set_file_system(file_system)
root = self.get_dom_for_path("/missing-streets/gazdagret/view-result")
results = root.findall("body/div[@id='no-ref-streets']")
self.assertEqual(len(results), 1)
def test_no_count_file(self) -> None:
"""Tests what happens when the count file is not there."""
ctx = test_context.make_test_context()
relations = areas.Relations(ctx)
relation = relations.get_relation("budafok")
hide_path = relation.get_files(
).get_housenumbers_additional_count_path()
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([hide_path])
ctx.set_file_system(file_system)
actual = wsgi.handle_main_housenr_additional_count(ctx, relation)
self.assertNotIn("42 housenumbers", actual.getvalue())
def test_view_result_txt_no_osm_streets(self) -> None:
"""Tests the txt output, no osm streets case."""
ctx = test_context.make_test_context()
relations = areas.Relations(ctx)
relation = relations.get_relation("gazdagret")
hide_path = relation.get_files().get_osm_streets_path()
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([hide_path])
self.ctx.set_file_system(file_system)
result = self.get_txt_for_path(
"/additional-streets/gazdagret/view-result.txt")
self.assertEqual(result, "No existing streets")
def test_no_cache(self) -> None:
"""Tests the case when there is no cache."""
ctx = test_context.make_test_context()
relations = areas.Relations(ctx)
relation = relations.get_relation("gazdagret")
cache.get_missing_housenumbers_html(ctx, relation)
cache_path = relation.get_files().get_housenumbers_htmlcache_path()
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([cache_path])
ctx.set_file_system(file_system)
self.assertFalse(
cache.is_missing_housenumbers_html_cached(ctx, relation))
def test_no_osm_housenumbers_well_formed(self) -> None:
"""Tests if the output is well-formed, no osm housenumbers case."""
ctx = test_context.make_test_context()
relations = areas.Relations(ctx)
relation = relations.get_relation("gazdagret")
hide_path = relation.get_files().get_osm_housenumbers_path()
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([hide_path])
self.ctx.set_file_system(file_system)
root = self.get_dom_for_path(
"/additional-housenumbers/gazdagret/view-result")
results = root.findall("body/div[@id='no-osm-housenumbers']")
self.assertEqual(len(results), 1)
def test_view_result_chkl_even_odd_split(self) -> None:
"""Tests the chkl output (even-odd streets)."""
hoursnumbers_ref = """Hamzsabégi út 1
Ref Name 1 1
Ref Name 1 2
Törökugrató utca 1 comment
Törökugrató utca 10
Törökugrató utca 11
Törökugrató utca 12
Törökugrató utca 2
Törökugrató utca 7
Tűzkő utca 1
Tűzkő utca 2
Tűzkő utca 9
Tűzkő utca 10
Tűzkő utca 12
Tűzkő utca 13
Tűzkő utca 14
Tűzkő utca 15
Tűzkő utca 16
Tűzkő utca 17
Tűzkő utca 18
Tűzkő utca 19
Tűzkő utca 20
Tűzkő utca 21
Tűzkő utca 22
Tűzkő utca 22
Tűzkő utca 24
Tűzkő utca 25
Tűzkő utca 26
Tűzkő utca 27
Tűzkő utca 28
Tűzkő utca 29
Tűzkő utca 30
Tűzkő utca 31
"""
hoursnumbers_ref_value = io.BytesIO()
hoursnumbers_ref_value.write(hoursnumbers_ref.encode("utf-8"))
hoursnumbers_ref_value.seek(0)
file_system = test_context.TestFileSystem()
hoursnumbers_ref_path = self.ctx.get_abspath("workdir/street-housenumbers-reference-gazdagret.lst")
file_system.set_files({hoursnumbers_ref_path: hoursnumbers_ref_value})
self.ctx.set_file_system(file_system)
result = self.get_txt_for_path("/missing-housenumbers/gazdagret/view-result.chkl")
expected = """[ ] Hamzsabégi út [1]
[ ] Törökugrató utca [7], [10]
[ ] Tűzkő utca [1, 13, 15, 17, 19, 21, 25, 27, 29, 31]
[ ] Tűzkő utca [2, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]"""
self.assertEqual(result, expected)
def test_incomplete_last_month(self) -> None:
"""Tests the case when we have no data for the last, incomplete month."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
# This would be the data for the current state of the last, incomplete month.
hide_path = ctx.get_abspath("workdir/stats/2020-05-10.count")
file_system = test_context.TestFileSystem()
file_system.set_hide_paths([hide_path])
ctx.set_file_system(file_system)
stats.handle_monthly_new(ctx, src_root, j)
monthly = j["monthly"]
# 1st element: 2019-05 start -> end
# No 2nd element, would be diff from last month end -> today
self.assertEqual(len(monthly), 1)
self.assertEqual(monthly[0], ["2019-05", 3799])
def test_ref_housenumbers_new(self) -> None:
"""Tests the case when ref_housenumbers is new, so the cache entry is old."""
ctx = test_context.make_test_context()
relations = areas.Relations(ctx)
relation = relations.get_relation("gazdagret")
cache.get_missing_housenumbers_html(ctx, relation)
cache_path = relation.get_files().get_housenumbers_htmlcache_path()
ref_housenumbers_path = relation.get_files().get_ref_housenumbers_path(
)
file_system = test_context.TestFileSystem()
mtimes = {
ref_housenumbers_path: os.path.getmtime(cache_path) + 1,
}
file_system.set_mtimes(mtimes)
ctx.set_file_system(file_system)
self.assertFalse(
cache.is_missing_housenumbers_html_cached(ctx, relation))
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
file_system = test_context.TestFileSystem()
stats_value = io.BytesIO()
stats_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("workdir/stats/stats.json"): stats_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
argv = ["", "--mode", "stats", "--no-overpass"]
buf = io.StringIO()
cron.main(argv, buf, ctx)
# Make sure that stats.json is updated without an error.
self.assertTrue(stats_value.tell())
self.assertNotIn("ERROR", buf.getvalue())
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
file_system = test_context.TestFileSystem()
old_citycount = b"""foo\t0
city1\t0
city2\t0
city3\t0
city4\t0
bar\t0
baz\t0
"""
old_citycount_value = io.BytesIO(old_citycount)
old_citycount_value.__setattr__("close", lambda: None)
new_citycount = b"""foo\t1000
city1\t1000
city2\t1000
city3\t1000
city4\t1000
bar\t2
baz\t2
"""
new_citycount_value = io.BytesIO(new_citycount)
new_citycount_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("workdir/stats/2020-04-10.citycount"): old_citycount_value,
ctx.get_abspath("workdir/stats/2020-05-10.citycount"): new_citycount_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
frequent_relations: Set[str] = {"foo", "bar"}
parse_access_log.check_top_edited_relations(ctx, frequent_relations)
self.assertIn("foo", frequent_relations)
self.assertIn("city1", frequent_relations)
self.assertIn("city2", frequent_relations)
self.assertIn("city3", frequent_relations)
self.assertIn("city4", frequent_relations)
self.assertNotIn("bar", frequent_relations)
self.assertNotIn("baz", frequent_relations)
def test_street_from_housenr_well_formed(self) -> None:
"""Tests if the output is well-formed when the street name comes from a housenr."""
file_system = test_context.TestFileSystem()
yamls_cache = {
"relations.yaml": {
"gh611": {
"osmrelation": 42,
},
},
"refcounty-names.yaml": {},
"refsettlement-names.yaml": {},
}
yamls_cache_value = io.BytesIO()
yamls_cache_value.write(json.dumps(yamls_cache).encode("utf-8"))
yamls_cache_value.seek(0)
file_system.set_files(
{self.ctx.get_abspath("data/yamls.cache"): yamls_cache_value})
self.ctx.set_file_system(file_system)
root = self.get_dom_for_path("/additional-streets/gh611/view-result")
results = root.findall("body/table")
self.assertEqual(len(results), 1)
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
file_system = test_context.TestFileSystem()
today_citycount = b"""budapest_01\t100
budapest_02\t200
\t42
"""
today_citycount_value = io.BytesIO(today_citycount)
today_citycount_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("workdir/stats/2020-05-10.citycount"):
today_citycount_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
src_root = ctx.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_topcities(ctx, src_root, j)
topcities = j["topcities"]
self.assertEqual(len(topcities), 2)
self.assertEqual(topcities[0], ("budapest_02", 190))
self.assertEqual(topcities[1], ("budapest_01", 90))
def test_no_csv(self) -> None:
"""Tests the case then the .csv is missing."""
ctx = test_context.make_test_context()
file_system = test_context.TestFileSystem()
today_topusers_value = io.BytesIO()
today_topusers_value.__setattr__("close", lambda: None)
today_usercount_value = io.BytesIO()
today_usercount_value.__setattr__("close", lambda: None)
file_system.set_hide_paths(
[ctx.get_abspath("workdir/stats/2020-05-10.csv")])
files = {
ctx.get_abspath("workdir/stats/2020-05-10.count"):
today_topusers_value,
ctx.get_abspath("workdir/stats/2020-05-10.citycount"):
today_usercount_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
cron.update_stats_topusers(ctx, "2020-05-10")
# No .csv, no .topusers or .usercount.
self.assertFalse(today_topusers_value.tell())
self.assertFalse(today_usercount_value.tell())
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
# For update_osm_streets().
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-streets-gazdagret.csv"),
# For update_osm_housenumbers().
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-housenumbers-gazdagret.csv"
),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
file_system = test_context.TestFileSystem()
yamls_cache = {
"relations.yaml": {
"gazdagret": {
"osmrelation": 2713748,
"refcounty": "01",
"refsettlement": "011",
},
},
"refcounty-names.yaml": {},
"refsettlement-names.yaml": {},
}
yamls_cache_value = io.BytesIO()
yamls_cache_value.__setattr__("close", lambda: None)
yamls_cache_value.write(util.to_bytes(json.dumps(yamls_cache)))
yamls_cache_value.seek(0)
osm_streets_value = io.BytesIO()
osm_streets_value.__setattr__("close", lambda: None)
osm_housenumbers_value = io.BytesIO()
osm_housenumbers_value.__setattr__("close", lambda: None)
ref_streets_value = io.BytesIO()
ref_streets_value.__setattr__("close", lambda: None)
ref_housenumbers_value = io.BytesIO()
ref_housenumbers_value.__setattr__("close", lambda: None)
missing_streets_value = io.BytesIO()
missing_streets_value.__setattr__("close", lambda: None)
missing_housenumbers_value = io.BytesIO()
missing_housenumbers_value.__setattr__("close", lambda: None)
additional_streets_value = io.BytesIO()
additional_streets_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("data/yamls.cache"):
yamls_cache_value,
ctx.get_abspath("workdir/streets-gazdagret.csv"):
osm_streets_value,
ctx.get_abspath("workdir/street-housenumbers-gazdagret.csv"):
osm_housenumbers_value,
ctx.get_abspath("workdir/streets-reference-gazdagret.lst"):
ref_streets_value,
ctx.get_abspath("workdir/street-housenumbers-reference-gazdagret.lst"):
ref_housenumbers_value,
ctx.get_abspath("workdir/gazdagret-streets.percent"):
missing_streets_value,
ctx.get_abspath("workdir/gazdagret.percent"):
missing_housenumbers_value,
ctx.get_abspath("workdir/gazdagret-additional-streets.count"):
additional_streets_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
relations = areas.Relations(ctx)
cron.our_main(ctx,
relations,
mode="relations",
update=True,
overpass=True)
# update_osm_streets() is called.
self.assertTrue(osm_streets_value.tell())
# update_osm_housenumbers() is called.
self.assertTrue(osm_housenumbers_value.tell())
# update_ref_streets() is called.
self.assertTrue(ref_streets_value.tell())
# update_ref_housenumbers() is called.
self.assertTrue(ref_housenumbers_value.tell())
# update_missing_streets() is called.
self.assertTrue(missing_streets_value.tell())
# update_missing_housenumbers() is called.
self.assertTrue(missing_housenumbers_value.tell())
# update_additional_streets() is called.
self.assertTrue(additional_streets_value.tell())
