def update_stats() -> None:
"""Performs the update of country-level stats."""
# Fetch house numbers for the whole country.
logging.info("update_stats: start, updating whole-country csv")
query = util.get_content(
config.get_abspath("data/street-housenumbers-hungary.txt"))
statedir = config.get_abspath("workdir/stats")
os.makedirs(statedir, exist_ok=True)
today = time.strftime("%Y-%m-%d")
csv_path = os.path.join(statedir, "%s.csv" % today)
retry = 0
while should_retry(retry):
if retry > 0:
logging.info("update_stats: try #%s", retry)
retry += 1
try:
overpass_sleep()
response = overpass_query.overpass_query(query)
with open(csv_path, "w") as stream:
stream.write(response)
break
except urllib.error.HTTPError as http_error:
logging.info("update_stats: http error: %s", str(http_error))
# Shell part.
logging.info("update_stats: executing the shell part")
subprocess.run([config.get_abspath("stats-daily.sh")], check=True)
logging.info("update_stats: end")
def handle_street_housenumbers(relations: helpers.Relations, request_uri: str) -> yattag.Doc:
"""Expected request_uri: e.g. /osm/street-housenumbers/ormezo/view-query."""
tokens = request_uri.split("/")
relation_name = tokens[-2]
action = tokens[-1]
relation = relations.get_relation(relation_name)
osmrelation = relation.get_config().get_osmrelation()
doc = yattag.Doc()
doc.asis(get_toolbar(relations, "street-housenumbers", relation_name, osmrelation).getvalue())
if action == "view-query":
with doc.tag("pre"):
doc.text(relation.get_osm_housenumbers_query())
elif action == "view-result":
with relation.get_files().get_osm_housenumbers_stream(mode="r") as sock:
table = util.tsv_to_list(sock)
doc.asis(util.html_table_from_list(table).getvalue())
elif action == "update-result":
query = relation.get_osm_housenumbers_query()
try:
relation.get_files().write_osm_housenumbers(overpass_query.overpass_query(query))
doc.text(_("Update successful: "))
link = "/osm/missing-housenumbers/" + relation_name + "/view-result"
doc.asis(util.gen_link(link, _("View missing house numbers")).getvalue())
except urllib.error.HTTPError as http_error:
doc.asis(util.handle_overpass_error(http_error).getvalue())
date = get_housenumbers_last_modified(relation)
doc.asis(get_footer(date).getvalue())
return doc
def test_happy(self) -> None:
"""Tests the happy path."""
with unittest.mock.patch('urllib.request.urlopen',
gen_urlopen("overpass-interpreter-happy")):
with open("tests/mock/overpass-interpreter-happy.request-data"
) as stream:
query = stream.read()
ret = overpass_query.overpass_query(query)
self.assertEqual(ret[:3], "@id")
def street_housenumbers_update_result_json(relations: areas.Relations, request_uri: str) -> str:
"""Expected request_uri: e.g. /osm/street-housenumbers/ormezo/update-result.json."""
tokens = request_uri.split("/")
relation_name = tokens[-2]
relation = relations.get_relation(relation_name)
query = relation.get_osm_housenumbers_query()
ret: Dict[str, str] = {}
try:
relation.get_files().write_osm_housenumbers(overpass_query.overpass_query(query))
ret["error"] = ""
except urllib.error.HTTPError as http_error:
ret["error"] = str(http_error)
return json.dumps(ret)
def handle_street_housenumbers(ctx: context.Context,
relations: areas.Relations,
request_uri: str) -> yattag.doc.Doc:
"""Expected request_uri: e.g. /osm/street-housenumbers/ormezo/view-query."""
tokens = request_uri.split("/")
relation_name = tokens[-2]
action = tokens[-1]
relation = relations.get_relation(relation_name)
osmrelation = relation.get_config().get_osmrelation()
doc = yattag.doc.Doc()
doc.asis(
webframe.get_toolbar(ctx, relations, "street-housenumbers",
relation_name, osmrelation).getvalue())
prefix = ctx.get_ini().get_uri_prefix()
if action == "view-query":
with doc.tag("pre"):
doc.text(relation.get_osm_housenumbers_query())
elif action == "update-result":
query = relation.get_osm_housenumbers_query()
buf, err = overpass_query.overpass_query(ctx, query)
if err:
doc.asis(util.handle_overpass_error(ctx, err).getvalue())
else:
relation.get_files().write_osm_housenumbers(ctx, buf)
doc.text(tr("Update successful: "))
link = prefix + "/missing-housenumbers/" + relation_name + "/view-result"
doc.asis(
util.gen_link(link,
tr("View missing house numbers")).getvalue())
else:
# assume view-result
if not ctx.get_file_system().path_exists(
relation.get_files().get_osm_housenumbers_path()):
with doc.tag("div", id="no-osm-housenumbers"):
doc.text(tr("No existing house numbers"))
else:
with relation.get_files().get_osm_housenumbers_csv_stream(
ctx) as sock:
doc.asis(
util.html_table_from_list(
util.tsv_to_list(sock)).getvalue())
date = get_housenumbers_last_modified(relation)
doc.asis(webframe.get_footer(date).getvalue())
return doc
def street_housenumbers_update_result_json(ctx: context.Context,
relations: areas.Relations,
request_uri: str) -> str:
"""Expected request_uri: e.g. /osm/street-housenumbers/ormezo/update-result.json."""
tokens = request_uri.split("/")
relation_name = tokens[-2]
relation = relations.get_relation(relation_name)
query = relation.get_osm_housenumbers_query()
ret: Dict[str, str] = {}
buf, err = overpass_query.overpass_query(ctx, query)
if err:
ret["error"] = err
else:
relation.get_files().write_osm_housenumbers(ctx, buf)
ret["error"] = ""
return json.dumps(ret)
def test_happy(self) -> None:
"""Tests the happy path."""
ctx = test_context.make_test_context()
routes: List[test_context.URLRoute] = [
test_context.URLRoute(
url="https://overpass-api.de/api/interpreter",
data_path="tests/network/overpass-happy.expected-data",
result_path="tests/network/overpass-happy.csv")
]
network = test_context.TestNetwork(routes)
ctx.set_network(network)
with open("tests/network/overpass-happy.expected-data") as stream:
query = stream.read()
buf, err = overpass_query.overpass_query(ctx, query)
self.assertEqual(buf[:3], "@id")
self.assertEqual(err, str())
def update_stats(overpass: bool) -> None:
"""Performs the update of country-level stats."""
# Fetch house numbers for the whole country.
info("update_stats: start, updating whole-country csv")
query = util.get_content(
config.get_abspath("data/street-housenumbers-hungary.txt")).decode(
"utf-8")
statedir = config.get_abspath("workdir/stats")
os.makedirs(statedir, exist_ok=True)
today = time.strftime("%Y-%m-%d")
csv_path = os.path.join(statedir, "%s.csv" % today)
if overpass:
retry = 0
while should_retry(retry):
if retry > 0:
info("update_stats: try #%s", retry)
retry += 1
try:
overpass_sleep()
response = overpass_query.overpass_query(query)
with open(csv_path, "w") as stream:
stream.write(response)
break
except urllib.error.HTTPError as http_error:
info("update_stats: http error: %s", str(http_error))
update_stats_count(today)
update_stats_topusers(today)
update_stats_refcount(statedir)
# Remove old CSV files as they are created daily and each is around 11M.
current_time = time.time()
for csv in glob.glob(os.path.join(statedir, "*.csv")):
creation_time = os.path.getmtime(csv)
if (current_time - creation_time) // (24 * 3600) >= 7:
os.unlink(csv)
info("update_stats: removed old %s", csv)
info("update_stats: generating json")
json_path = os.path.join(statedir, "stats.json")
with open(json_path, "w") as stream:
stats.generate_json(statedir, stream)
info("update_stats: end")
def update_stats(ctx: context.Context, overpass: bool) -> None:
"""Performs the update of country-level stats."""
# Fetch house numbers for the whole country.
info("update_stats: start, updating whole-country csv")
query = util.from_bytes(
util.get_content(
ctx.get_abspath("data/street-housenumbers-hungary.txt")))
statedir = ctx.get_abspath("workdir/stats")
os.makedirs(statedir, exist_ok=True)
today = time.strftime("%Y-%m-%d")
csv_path = os.path.join(statedir, "%s.csv" % today)
if overpass:
retry = 0
while should_retry(retry):
if retry > 0:
info("update_stats: try #%s", retry)
retry += 1
overpass_sleep(ctx)
response, err = overpass_query.overpass_query(ctx, query)
if err:
info("update_stats: http error: %s", err)
continue
with open(csv_path, "wb") as stream:
stream.write(util.to_bytes(response))
break
update_stats_count(ctx, today)
update_stats_topusers(ctx, today)
update_stats_refcount(ctx, statedir)
# Remove old CSV files as they are created daily and each is around 11M.
current_time = time.time()
for csv in glob.glob(os.path.join(statedir, "*.csv")):
creation_time = os.path.getmtime(csv)
if (current_time - creation_time) // (24 * 3600) >= 7:
os.unlink(csv)
info("update_stats: removed old %s", csv)
info("update_stats: generating json")
json_path = os.path.join(statedir, "stats.json")
with ctx.get_file_system().open(json_path, "wb") as stream:
stats.generate_json(ctx, statedir, stream)
info("update_stats: end")
def update_osm_housenumbers(relations: areas.Relations) -> None:
"""Update the OSM housenumber list of all relations."""
for relation_name in relations.get_active_names():
logging.info("update_osm_housenumbers: start: %s", relation_name)
retry = 0
while should_retry(retry):
if retry > 0:
logging.info("update_osm_housenumbers: try #%s", retry)
retry += 1
try:
overpass_sleep()
relation = relations.get_relation(relation_name)
query = relation.get_osm_housenumbers_query()
relation.get_files().write_osm_housenumbers(overpass_query.overpass_query(query))
break
except urllib.error.HTTPError as http_error:
logging.info("update_osm_housenumbers: http error: %s", str(http_error))
logging.info("update_osm_housenumbers: end: %s", relation_name)
def handle_streets(relations: areas.Relations,
request_uri: str) -> yattag.doc.Doc:
"""Expected request_uri: e.g. /osm/streets/ormezo/view-query."""
tokens = request_uri.split("/")
relation_name = tokens[-2]
action = tokens[-1]
relation = relations.get_relation(relation_name)
osmrelation = relation.get_config().get_osmrelation()
doc = yattag.doc.Doc()
doc.asis(
webframe.get_toolbar(relations, "streets", relation_name,
osmrelation).getvalue())
prefix = config.Config.get_uri_prefix()
if action == "view-query":
with doc.tag("pre"):
doc.text(relation.get_osm_streets_query())
elif action == "update-result":
query = relation.get_osm_streets_query()
try:
relation.get_files().write_osm_streets(
overpass_query.overpass_query(query))
streets = relation.get_config().should_check_missing_streets()
if streets != "only":
doc.text(_("Update successful: "))
link = prefix + "/missing-housenumbers/" + relation_name + "/view-result"
doc.asis(
util.gen_link(link,
_("View missing house numbers")).getvalue())
else:
doc.text(_("Update successful."))
except urllib.error.HTTPError as http_error:
doc.asis(util.handle_overpass_error(http_error).getvalue())
else:
# assume view-result
with relation.get_files().get_osm_streets_stream("r") as sock:
table = util.tsv_to_list(sock)
doc.asis(util.html_table_from_list(table).getvalue())
doc.asis(
webframe.get_footer(get_streets_last_modified(relation)).getvalue())
return doc
def update_osm_housenumbers(relations: areas.Relations, update: bool) -> None:
"""Update the OSM housenumber list of all relations."""
for relation_name in relations.get_active_names():
relation = relations.get_relation(relation_name)
if not update and os.path.exists(relation.get_files().get_osm_housenumbers_path()):
continue
info("update_osm_housenumbers: start: %s", relation_name)
retry = 0
while should_retry(retry):
if retry > 0:
info("update_osm_housenumbers: try #%s", retry)
retry += 1
try:
overpass_sleep()
query = relation.get_osm_housenumbers_query()
relation.get_files().write_osm_housenumbers(overpass_query.overpass_query(query))
break
except urllib.error.HTTPError as http_error:
info("update_osm_housenumbers: http error: %s", str(http_error))
info("update_osm_housenumbers: end: %s", relation_name)
def update_osm_housenumbers(ctx: context.Context, relations: areas.Relations,
update: bool) -> None:
"""Update the OSM housenumber list of all relations."""
for relation_name in relations.get_active_names():
relation = relations.get_relation(relation_name)
if not update and os.path.exists(
relation.get_files().get_osm_housenumbers_path()):
continue
info("update_osm_housenumbers: start: %s", relation_name)
retry = 0
while should_retry(retry):
if retry > 0:
info("update_osm_housenumbers: try #%s", retry)
retry += 1
overpass_sleep(ctx)
query = relation.get_osm_housenumbers_query()
buf, err = overpass_query.overpass_query(ctx, query)
if err:
info("update_osm_housenumbers: http error: %s", err)
continue
relation.get_files().write_osm_housenumbers(ctx, buf)
break
info("update_osm_housenumbers: end: %s", relation_name)