def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-stats.csv"),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
# Create a CSV that is definitely old enough to be removed.
old_path = ctx.get_abspath("workdir/stats/old.csv")
create_old_file(old_path)
today = time.strftime("%Y-%m-%d")
path = ctx.get_abspath("workdir/stats/%s.csv" % today)
cron.update_stats(ctx, overpass=True)
actual = util.get_content(path)
self.assertEqual(actual,
util.get_content("tests/network/overpass-stats.csv"))
# Make sure that the old CSV is removed.
self.assertFalse(os.path.exists(old_path))
with open(ctx.get_abspath("workdir/stats/ref.count"), "r") as stream:
num_ref = int(stream.read().strip())
self.assertEqual(num_ref, 300)
def test_stats(self) -> None:
"""Tests the stats path."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-stats.csv"),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
file_system = test_context.TestFileSystem()
stats_value = io.BytesIO()
stats_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("workdir/stats/stats.json"): stats_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
relations = areas.Relations(ctx)
cron.our_main(ctx,
relations,
mode="stats",
update=False,
overpass=True)
self.assertTrue(stats_value.tell())
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-streets-gazdagret.csv"),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
relations = areas.Relations(ctx)
for relation_name in relations.get_active_names():
if relation_name != "gazdagret":
relations.get_relation(relation_name).get_config().set_active(
False)
expected = util.get_content(relations.get_workdir(),
"streets-gazdagret.csv")
path = os.path.join(relations.get_workdir(), "streets-gazdagret.csv")
os.unlink(path)
cron.update_osm_streets(ctx, relations, update=True)
mtime = os.path.getmtime(path)
cron.update_osm_streets(ctx, relations, update=False)
self.assertEqual(os.path.getmtime(path), mtime)
actual = util.get_content(relations.get_workdir(),
"streets-gazdagret.csv")
self.assertEqual(actual, expected)
def test_http_error(self) -> None:
"""Tests the case when we keep getting HTTP errors."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path=""),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
relations = areas.Relations(ctx)
for relation_name in relations.get_active_names():
if relation_name != "gazdagret":
relations.get_relation(relation_name).get_config().set_active(
False)
expected = util.get_content(relations.get_workdir(),
"streets-gazdagret.csv")
cron.update_osm_streets(ctx, relations, update=True)
# Make sure that in case we keep getting errors we give up at some stage and
# leave the last state unchanged.
actual = util.get_content(relations.get_workdir(),
"streets-gazdagret.csv")
self.assertEqual(actual, expected)
def test_need_sleep(self) -> None:
"""Tests the case when sleep is needed."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-wait.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt")
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
ctx.set_time(test_context.make_test_time())
cron.overpass_sleep(ctx)
test_time = cast(test_context.TestTime, ctx.get_time())
self.assertEqual(test_time.get_sleep(), 12)
def test_no_overpass(self) -> None:
"""Tests the case when we don't call overpass."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-wait.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt")
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
cron.update_stats(ctx, overpass=False)
test_time = cast(test_context.TestTime, ctx.get_time())
self.assertEqual(test_time.get_sleep(), 0)
def test_update_result_error_well_formed(self) -> None:
"""Tests if the update-result output on error is well-formed."""
routes: List[test_context.URLRoute] = [
test_context.URLRoute(url="https://overpass-api.de/api/interpreter",
data_path="",
result_path=""),
]
network = test_context.TestNetwork(routes)
self.ctx.set_network(network)
root = self.get_dom_for_path("/street-housenumbers/gazdagret/update-result")
results = root.findall("body/div[@id='overpass-error']")
self.assertTrue(results)
def test_update_result_well_formed(self) -> None:
"""Tests if the update-result output is well-formed."""
routes: List[test_context.URLRoute] = [
test_context.URLRoute(url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-housenumbers-gazdagret.csv"),
]
network = test_context.TestNetwork(routes)
self.ctx.set_network(network)
root = self.get_dom_for_path("/street-housenumbers/gazdagret/update-result")
results = root.findall("body")
self.assertEqual(len(results), 1)
def test_wait_negative(self) -> None:
"""Tests the wait for negative amount path."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-wait-negative.txt")
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
self.assertEqual(overpass_query.overpass_query_need_sleep(ctx), 1)
def test_update_result_json_error(self) -> None:
"""Tests if the update-result json output on error is well-formed."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="")
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
root = self.get_json_for_path(ctx,
"/streets/gazdagret/update-result.json")
self.assertIn("error", root)
def test_update_result_json(self) -> None:
"""Tests if the update-result output is well-formed."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-housenumbers-gazdagret.csv"
)
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
root = self.get_json_for_path(
ctx, "/street-housenumbers/gazdagret/update-result.json")
self.assertEqual(root["error"], "")
def test_update_result_missing_streets_well_formed(self) -> None:
"""
Tests if the update-result output is well-formed for should_check_missing_streets() ==
"only".
"""
routes: List[test_context.URLRoute] = [
test_context.URLRoute(url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-streets-ujbuda.csv"),
]
network = test_context.TestNetwork(routes)
self.ctx.set_network(network)
root = self.get_dom_for_path("/streets/ujbuda/update-result")
results = root.findall("body")
self.assertEqual(len(results), 1)
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="tests/network/overpass-happy.expected-data",
result_path="tests/network/overpass-happy.csv")
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
with open("tests/network/overpass-happy.expected-data") as stream:
query = stream.read()
buf, err = overpass_query.overpass_query(ctx, query)
self.assertEqual(buf[:3], "@id")
self.assertEqual(err, str())
def test_http_error(self) -> None:
"""Tests the case when we keep getting HTTP errors."""
ctx = test_context.make_test_context()
ctx.set_time(test_context.make_test_time())
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
old_mtime = ctx.get_file_system().getmtime(
ctx.get_abspath("workdir/stats/stats.json"))
cron.update_stats(ctx, overpass=True)
new_mtime = ctx.get_file_system().getmtime(
ctx.get_abspath("workdir/stats/stats.json"))
self.assertGreater(new_mtime, old_mtime)
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
# For update_osm_streets().
test_context.URLRoute(
url="https://overpass-api.de/api/status",
data_path="",
result_path="tests/network/overpass-status-happy.txt"),
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-streets-gazdagret.csv"),
# For update_osm_housenumbers().
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="",
result_path="tests/network/overpass-housenumbers-gazdagret.csv"
),
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
file_system = test_context.TestFileSystem()
yamls_cache = {
"relations.yaml": {
"gazdagret": {
"osmrelation": 2713748,
"refcounty": "01",
"refsettlement": "011",
},
},
"refcounty-names.yaml": {},
"refsettlement-names.yaml": {},
}
yamls_cache_value = io.BytesIO()
yamls_cache_value.__setattr__("close", lambda: None)
yamls_cache_value.write(util.to_bytes(json.dumps(yamls_cache)))
yamls_cache_value.seek(0)
osm_streets_value = io.BytesIO()
osm_streets_value.__setattr__("close", lambda: None)
osm_housenumbers_value = io.BytesIO()
osm_housenumbers_value.__setattr__("close", lambda: None)
ref_streets_value = io.BytesIO()
ref_streets_value.__setattr__("close", lambda: None)
ref_housenumbers_value = io.BytesIO()
ref_housenumbers_value.__setattr__("close", lambda: None)
missing_streets_value = io.BytesIO()
missing_streets_value.__setattr__("close", lambda: None)
missing_housenumbers_value = io.BytesIO()
missing_housenumbers_value.__setattr__("close", lambda: None)
additional_streets_value = io.BytesIO()
additional_streets_value.__setattr__("close", lambda: None)
files = {
ctx.get_abspath("data/yamls.cache"):
yamls_cache_value,
ctx.get_abspath("workdir/streets-gazdagret.csv"):
osm_streets_value,
ctx.get_abspath("workdir/street-housenumbers-gazdagret.csv"):
osm_housenumbers_value,
ctx.get_abspath("workdir/streets-reference-gazdagret.lst"):
ref_streets_value,
ctx.get_abspath("workdir/street-housenumbers-reference-gazdagret.lst"):
ref_housenumbers_value,
ctx.get_abspath("workdir/gazdagret-streets.percent"):
missing_streets_value,
ctx.get_abspath("workdir/gazdagret.percent"):
missing_housenumbers_value,
ctx.get_abspath("workdir/gazdagret-additional-streets.count"):
additional_streets_value,
}
file_system.set_files(files)
ctx.set_file_system(file_system)
relations = areas.Relations(ctx)
cron.our_main(ctx,
relations,
mode="relations",
update=True,
overpass=True)
# update_osm_streets() is called.
self.assertTrue(osm_streets_value.tell())
# update_osm_housenumbers() is called.
self.assertTrue(osm_housenumbers_value.tell())
# update_ref_streets() is called.
self.assertTrue(ref_streets_value.tell())
# update_ref_housenumbers() is called.
self.assertTrue(ref_housenumbers_value.tell())
# update_missing_streets() is called.
self.assertTrue(missing_streets_value.tell())
# update_missing_housenumbers() is called.
self.assertTrue(missing_housenumbers_value.tell())
# update_additional_streets() is called.
self.assertTrue(additional_streets_value.tell())
