def test_stats(self) -> None: """Tests the stats path.""" ctx = test_context.make_test_context() routes: List[test_context.URLRoute] = [ test_context.URLRoute( url="https://overpass-api.de/api/status", data_path="", result_path="tests/network/overpass-status-happy.txt"), test_context.URLRoute( url="https://overpass-api.de/api/interpreter", data_path="", result_path="tests/network/overpass-stats.csv"), ] network = test_context.TestNetwork(routes) ctx.set_network(network) file_system = test_context.TestFileSystem() stats_value = io.BytesIO() stats_value.__setattr__("close", lambda: None) files = { ctx.get_abspath("workdir/stats/stats.json"): stats_value, } file_system.set_files(files) ctx.set_file_system(file_system) relations = areas.Relations(ctx) cron.our_main(ctx, relations, mode="stats", update=False, overpass=True) self.assertTrue(stats_value.tell())
def test_happy(self) -> None: """Tests tha happy path.""" ctx = test_context.make_test_context() file_system = test_context.TestFileSystem() today_csv = util.to_bytes( """addr:postcode addr:city addr:street addr:housenumber @user 7677 Orfű Dollár utca 1 mgpx """) today_csv_value = io.BytesIO(today_csv) today_csv_value.__setattr__("close", lambda: None) today_topusers_value = io.BytesIO() today_topusers_value.__setattr__("close", lambda: None) today_usercount_value = io.BytesIO() today_usercount_value.__setattr__("close", lambda: None) files = { ctx.get_abspath("workdir/stats/2020-05-10.csv"): today_csv_value, ctx.get_abspath("workdir/stats/2020-05-10.topusers"): today_topusers_value, ctx.get_abspath("workdir/stats/2020-05-10.usercount"): today_usercount_value, } file_system.set_files(files) ctx.set_file_system(file_system) cron.update_stats_topusers(ctx, "2020-05-10") self.assertTrue(today_topusers_value.tell()) self.assertTrue(today_usercount_value.tell())
def test_view_result_txt_no_ref_streets(self) -> None: """Tests the txt output, no ref streets case.""" relations = areas.Relations(test_context.make_test_context()) relation = relations.get_relation("gazdagret") hide_path = relation.get_files().get_ref_streets_path() file_system = test_context.TestFileSystem() file_system.set_hide_paths([hide_path]) self.ctx.set_file_system(file_system) result = self.get_txt_for_path("/missing-streets/gazdagret/view-result.txt") self.assertEqual(result, "No reference streets")
def test_view_result_txt_no_osm_housenumbers(self) -> None: """Tests the txt output, no osm housenumbers case.""" relations = areas.Relations(test_context.make_test_context()) relation = relations.get_relation("gazdagret") hide_path = relation.get_files().get_osm_housenumbers_path() file_system = test_context.TestFileSystem() file_system.set_hide_paths([hide_path]) self.ctx.set_file_system(file_system) result = self.get_txt_for_path("/missing-housenumbers/gazdagret/view-result.txt") self.assertEqual(result, "No existing house numbers")
def test_new_missing(self) -> None: """Tests the case when the new path is missing.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) file_system = test_context.TestFileSystem() src_root = ctx.get_abspath("workdir/stats") file_system.set_hide_paths( [os.path.join(src_root, "2020-05-10.citycount")]) ctx.set_file_system(file_system) ret = stats.get_topcities(ctx, src_root) self.assertEqual(ret, [])
def test_no_osm_sreets(self) -> None: """Tests error handling when osm street list is missing for a relation.""" relations = areas.Relations(test_context.make_test_context()) relation = relations.get_relation("gazdagret") hide_path = relation.get_files().get_osm_streets_path() file_system = test_context.TestFileSystem() file_system.set_hide_paths([hide_path]) self.ctx.set_file_system(file_system) root = self.get_dom_for_path("/housenumber-stats/hungary/invalid-relations") results = root.findall("body") self.assertNotEqual(results, [])
def test_no_ref_streets_well_formed(self) -> None: """Tests if the output is well-formed, no ref streets case.""" relations = areas.Relations(test_context.make_test_context()) relation = relations.get_relation("gazdagret") hide_path = relation.get_files().get_ref_streets_path() file_system = test_context.TestFileSystem() file_system.set_hide_paths([hide_path]) self.ctx.set_file_system(file_system) root = self.get_dom_for_path("/missing-streets/gazdagret/view-result") results = root.findall("body/div[@id='no-ref-streets']") self.assertEqual(len(results), 1)
def test_no_count_file(self) -> None: """Tests what happens when the count file is not there.""" ctx = test_context.make_test_context() relations = areas.Relations(ctx) relation = relations.get_relation("budafok") hide_path = relation.get_files( ).get_housenumbers_additional_count_path() file_system = test_context.TestFileSystem() file_system.set_hide_paths([hide_path]) ctx.set_file_system(file_system) actual = wsgi.handle_main_housenr_additional_count(ctx, relation) self.assertNotIn("42 housenumbers", actual.getvalue())
def test_view_result_txt_no_osm_streets(self) -> None: """Tests the txt output, no osm streets case.""" ctx = test_context.make_test_context() relations = areas.Relations(ctx) relation = relations.get_relation("gazdagret") hide_path = relation.get_files().get_osm_streets_path() file_system = test_context.TestFileSystem() file_system.set_hide_paths([hide_path]) self.ctx.set_file_system(file_system) result = self.get_txt_for_path( "/additional-streets/gazdagret/view-result.txt") self.assertEqual(result, "No existing streets")
def test_no_cache(self) -> None: """Tests the case when there is no cache.""" ctx = test_context.make_test_context() relations = areas.Relations(ctx) relation = relations.get_relation("gazdagret") cache.get_missing_housenumbers_html(ctx, relation) cache_path = relation.get_files().get_housenumbers_htmlcache_path() file_system = test_context.TestFileSystem() file_system.set_hide_paths([cache_path]) ctx.set_file_system(file_system) self.assertFalse( cache.is_missing_housenumbers_html_cached(ctx, relation))
def test_no_osm_housenumbers_well_formed(self) -> None: """Tests if the output is well-formed, no osm housenumbers case.""" ctx = test_context.make_test_context() relations = areas.Relations(ctx) relation = relations.get_relation("gazdagret") hide_path = relation.get_files().get_osm_housenumbers_path() file_system = test_context.TestFileSystem() file_system.set_hide_paths([hide_path]) self.ctx.set_file_system(file_system) root = self.get_dom_for_path( "/additional-housenumbers/gazdagret/view-result") results = root.findall("body/div[@id='no-osm-housenumbers']") self.assertEqual(len(results), 1)
def test_view_result_chkl_even_odd_split(self) -> None: """Tests the chkl output (even-odd streets).""" hoursnumbers_ref = """Hamzsabégi út 1 Ref Name 1 1 Ref Name 1 2 Törökugrató utca 1 comment Törökugrató utca 10 Törökugrató utca 11 Törökugrató utca 12 Törökugrató utca 2 Törökugrató utca 7 Tűzkő utca 1 Tűzkő utca 2 Tűzkő utca 9 Tűzkő utca 10 Tűzkő utca 12 Tűzkő utca 13 Tűzkő utca 14 Tűzkő utca 15 Tűzkő utca 16 Tűzkő utca 17 Tűzkő utca 18 Tűzkő utca 19 Tűzkő utca 20 Tűzkő utca 21 Tűzkő utca 22 Tűzkő utca 22 Tűzkő utca 24 Tűzkő utca 25 Tűzkő utca 26 Tűzkő utca 27 Tűzkő utca 28 Tűzkő utca 29 Tűzkő utca 30 Tűzkő utca 31 """ hoursnumbers_ref_value = io.BytesIO() hoursnumbers_ref_value.write(hoursnumbers_ref.encode("utf-8")) hoursnumbers_ref_value.seek(0) file_system = test_context.TestFileSystem() hoursnumbers_ref_path = self.ctx.get_abspath("workdir/street-housenumbers-reference-gazdagret.lst") file_system.set_files({hoursnumbers_ref_path: hoursnumbers_ref_value}) self.ctx.set_file_system(file_system) result = self.get_txt_for_path("/missing-housenumbers/gazdagret/view-result.chkl") expected = """[ ] Hamzsabégi út [1] [ ] Törökugrató utca [7], [10] [ ] Tűzkő utca [1, 13, 15, 17, 19, 21, 25, 27, 29, 31] [ ] Tűzkő utca [2, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]""" self.assertEqual(result, expected)
def test_incomplete_last_month(self) -> None: """Tests the case when we have no data for the last, incomplete month.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} # This would be the data for the current state of the last, incomplete month. hide_path = ctx.get_abspath("workdir/stats/2020-05-10.count") file_system = test_context.TestFileSystem() file_system.set_hide_paths([hide_path]) ctx.set_file_system(file_system) stats.handle_monthly_new(ctx, src_root, j) monthly = j["monthly"] # 1st element: 2019-05 start -> end # No 2nd element, would be diff from last month end -> today self.assertEqual(len(monthly), 1) self.assertEqual(monthly[0], ["2019-05", 3799])
def test_ref_housenumbers_new(self) -> None: """Tests the case when ref_housenumbers is new, so the cache entry is old.""" ctx = test_context.make_test_context() relations = areas.Relations(ctx) relation = relations.get_relation("gazdagret") cache.get_missing_housenumbers_html(ctx, relation) cache_path = relation.get_files().get_housenumbers_htmlcache_path() ref_housenumbers_path = relation.get_files().get_ref_housenumbers_path( ) file_system = test_context.TestFileSystem() mtimes = { ref_housenumbers_path: os.path.getmtime(cache_path) + 1, } file_system.set_mtimes(mtimes) ctx.set_file_system(file_system) self.assertFalse( cache.is_missing_housenumbers_html_cached(ctx, relation))
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() file_system = test_context.TestFileSystem() stats_value = io.BytesIO() stats_value.__setattr__("close", lambda: None) files = { ctx.get_abspath("workdir/stats/stats.json"): stats_value, } file_system.set_files(files) ctx.set_file_system(file_system) argv = ["", "--mode", "stats", "--no-overpass"] buf = io.StringIO() cron.main(argv, buf, ctx) # Make sure that stats.json is updated without an error. self.assertTrue(stats_value.tell()) self.assertNotIn("ERROR", buf.getvalue())
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) file_system = test_context.TestFileSystem() old_citycount = b"""foo\t0 city1\t0 city2\t0 city3\t0 city4\t0 bar\t0 baz\t0 """ old_citycount_value = io.BytesIO(old_citycount) old_citycount_value.__setattr__("close", lambda: None) new_citycount = b"""foo\t1000 city1\t1000 city2\t1000 city3\t1000 city4\t1000 bar\t2 baz\t2 """ new_citycount_value = io.BytesIO(new_citycount) new_citycount_value.__setattr__("close", lambda: None) files = { ctx.get_abspath("workdir/stats/2020-04-10.citycount"): old_citycount_value, ctx.get_abspath("workdir/stats/2020-05-10.citycount"): new_citycount_value, } file_system.set_files(files) ctx.set_file_system(file_system) frequent_relations: Set[str] = {"foo", "bar"} parse_access_log.check_top_edited_relations(ctx, frequent_relations) self.assertIn("foo", frequent_relations) self.assertIn("city1", frequent_relations) self.assertIn("city2", frequent_relations) self.assertIn("city3", frequent_relations) self.assertIn("city4", frequent_relations) self.assertNotIn("bar", frequent_relations) self.assertNotIn("baz", frequent_relations)
def test_street_from_housenr_well_formed(self) -> None: """Tests if the output is well-formed when the street name comes from a housenr.""" file_system = test_context.TestFileSystem() yamls_cache = { "relations.yaml": { "gh611": { "osmrelation": 42, }, }, "refcounty-names.yaml": {}, "refsettlement-names.yaml": {}, } yamls_cache_value = io.BytesIO() yamls_cache_value.write(json.dumps(yamls_cache).encode("utf-8")) yamls_cache_value.seek(0) file_system.set_files( {self.ctx.get_abspath("data/yamls.cache"): yamls_cache_value}) self.ctx.set_file_system(file_system) root = self.get_dom_for_path("/additional-streets/gh611/view-result") results = root.findall("body/table") self.assertEqual(len(results), 1)
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() ctx.set_time(test_context.make_test_time()) file_system = test_context.TestFileSystem() today_citycount = b"""budapest_01\t100 budapest_02\t200 \t42 """ today_citycount_value = io.BytesIO(today_citycount) today_citycount_value.__setattr__("close", lambda: None) files = { ctx.get_abspath("workdir/stats/2020-05-10.citycount"): today_citycount_value, } file_system.set_files(files) ctx.set_file_system(file_system) src_root = ctx.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_topcities(ctx, src_root, j) topcities = j["topcities"] self.assertEqual(len(topcities), 2) self.assertEqual(topcities[0], ("budapest_02", 190)) self.assertEqual(topcities[1], ("budapest_01", 90))
def test_no_csv(self) -> None: """Tests the case then the .csv is missing.""" ctx = test_context.make_test_context() file_system = test_context.TestFileSystem() today_topusers_value = io.BytesIO() today_topusers_value.__setattr__("close", lambda: None) today_usercount_value = io.BytesIO() today_usercount_value.__setattr__("close", lambda: None) file_system.set_hide_paths( [ctx.get_abspath("workdir/stats/2020-05-10.csv")]) files = { ctx.get_abspath("workdir/stats/2020-05-10.count"): today_topusers_value, ctx.get_abspath("workdir/stats/2020-05-10.citycount"): today_usercount_value, } file_system.set_files(files) ctx.set_file_system(file_system) cron.update_stats_topusers(ctx, "2020-05-10") # No .csv, no .topusers or .usercount. self.assertFalse(today_topusers_value.tell()) self.assertFalse(today_usercount_value.tell())
def test_happy(self) -> None: """Tests the happy path.""" ctx = test_context.make_test_context() routes: List[test_context.URLRoute] = [ # For update_osm_streets(). test_context.URLRoute( url="https://overpass-api.de/api/status", data_path="", result_path="tests/network/overpass-status-happy.txt"), test_context.URLRoute( url="https://overpass-api.de/api/interpreter", data_path="", result_path="tests/network/overpass-streets-gazdagret.csv"), # For update_osm_housenumbers(). test_context.URLRoute( url="https://overpass-api.de/api/interpreter", data_path="", result_path="tests/network/overpass-housenumbers-gazdagret.csv" ), ] network = test_context.TestNetwork(routes) ctx.set_network(network) file_system = test_context.TestFileSystem() yamls_cache = { "relations.yaml": { "gazdagret": { "osmrelation": 2713748, "refcounty": "01", "refsettlement": "011", }, }, "refcounty-names.yaml": {}, "refsettlement-names.yaml": {}, } yamls_cache_value = io.BytesIO() yamls_cache_value.__setattr__("close", lambda: None) yamls_cache_value.write(util.to_bytes(json.dumps(yamls_cache))) yamls_cache_value.seek(0) osm_streets_value = io.BytesIO() osm_streets_value.__setattr__("close", lambda: None) osm_housenumbers_value = io.BytesIO() osm_housenumbers_value.__setattr__("close", lambda: None) ref_streets_value = io.BytesIO() ref_streets_value.__setattr__("close", lambda: None) ref_housenumbers_value = io.BytesIO() ref_housenumbers_value.__setattr__("close", lambda: None) missing_streets_value = io.BytesIO() missing_streets_value.__setattr__("close", lambda: None) missing_housenumbers_value = io.BytesIO() missing_housenumbers_value.__setattr__("close", lambda: None) additional_streets_value = io.BytesIO() additional_streets_value.__setattr__("close", lambda: None) files = { ctx.get_abspath("data/yamls.cache"): yamls_cache_value, ctx.get_abspath("workdir/streets-gazdagret.csv"): osm_streets_value, ctx.get_abspath("workdir/street-housenumbers-gazdagret.csv"): osm_housenumbers_value, ctx.get_abspath("workdir/streets-reference-gazdagret.lst"): ref_streets_value, ctx.get_abspath("workdir/street-housenumbers-reference-gazdagret.lst"): ref_housenumbers_value, ctx.get_abspath("workdir/gazdagret-streets.percent"): missing_streets_value, ctx.get_abspath("workdir/gazdagret.percent"): missing_housenumbers_value, ctx.get_abspath("workdir/gazdagret-additional-streets.count"): additional_streets_value, } file_system.set_files(files) ctx.set_file_system(file_system) relations = areas.Relations(ctx) cron.our_main(ctx, relations, mode="relations", update=True, overpass=True) # update_osm_streets() is called. self.assertTrue(osm_streets_value.tell()) # update_osm_housenumbers() is called. self.assertTrue(osm_housenumbers_value.tell()) # update_ref_streets() is called. self.assertTrue(ref_streets_value.tell()) # update_ref_housenumbers() is called. self.assertTrue(ref_housenumbers_value.tell()) # update_missing_streets() is called. self.assertTrue(missing_streets_value.tell()) # update_missing_housenumbers() is called. self.assertTrue(missing_housenumbers_value.tell()) # update_additional_streets() is called. self.assertTrue(additional_streets_value.tell())