def handle_static(
request_uri: str) -> Tuple[bytes, str, List[Tuple[str, str]]]:
"""Handles serving static content."""
tokens = request_uri.split("/")
path = tokens[-1]
extra_headers: List[Tuple[str, str]] = []
if request_uri.endswith(".js"):
content_type = "application/x-javascript"
content = util.get_content(config.Config.get_workdir(), path,
extra_headers)
return content, content_type, extra_headers
if request_uri.endswith(".css"):
content_type = "text/css"
content = util.get_content(config.get_abspath("static"), path,
extra_headers)
return content, content_type, extra_headers
if request_uri.endswith(".json"):
content_type = "application/json"
content = util.get_content(
os.path.join(config.Config.get_workdir(), "stats"), path,
extra_headers)
return content, content_type, extra_headers
if request_uri.endswith(".ico"):
content_type = "image/x-icon"
content = util.get_content(config.get_abspath(""), path, extra_headers)
return content, content_type, extra_headers
return bytes(), "", extra_headers
def update_stats() -> None:
"""Performs the update of country-level stats."""
# Fetch house numbers for the whole country.
logging.info("update_stats: start, updating whole-country csv")
query = util.get_content(
config.get_abspath("data/street-housenumbers-hungary.txt"))
statedir = config.get_abspath("workdir/stats")
os.makedirs(statedir, exist_ok=True)
today = time.strftime("%Y-%m-%d")
csv_path = os.path.join(statedir, "%s.csv" % today)
retry = 0
while should_retry(retry):
if retry > 0:
logging.info("update_stats: try #%s", retry)
retry += 1
try:
overpass_sleep()
response = overpass_query.overpass_query(query)
with open(csv_path, "w") as stream:
stream.write(response)
break
except urllib.error.HTTPError as http_error:
logging.info("update_stats: http error: %s", str(http_error))
# Shell part.
logging.info("update_stats: executing the shell part")
subprocess.run([config.get_abspath("stats-daily.sh")], check=True)
logging.info("update_stats: end")
def test_happy(self) -> None:
"""Tests the happy path."""
expected = util.get_content(config.get_abspath("workdir/street-housenumbers-reference-gazdagret.lst"))
argv = ["", "gazdagret"]
with unittest.mock.patch('sys.argv', argv):
get_reference_housenumbers.main()
actual = util.get_content(config.get_abspath("workdir/street-housenumbers-reference-gazdagret.lst"))
self.assertEqual(actual, expected)
def update_stats_count(today: str) -> None:
"""Counts the # of all house numbers as of today."""
statedir = config.get_abspath("workdir/stats")
csv_path = os.path.join(statedir, "%s.csv" % today)
count_path = os.path.join(statedir, "%s.count" % today)
city_count_path = os.path.join(statedir, "%s.citycount" % today)
house_numbers = set()
cities: Dict[str, int] = {}
first = True
with open(csv_path, "r") as stream:
for line in stream.readlines():
if first:
# Ignore the oneliner header.
first = False
continue
cells = line.split("\t")
# Ignore last column, which is the user who touched the object last.
house_numbers.add("\t".join(cells[:4]))
city_key = util.get_city_key(cells[0], cells[1])
if city_key in cities:
cities[city_key] += 1
else:
cities[city_key] = 1
with open(count_path, "w") as stream:
house_numbers_len = str(len(house_numbers))
stream.write(house_numbers_len + "\n")
with open(city_count_path, "w") as stream:
for key, value in cities.items():
stream.write(key + "\t" + str(value) + "\n")
def write_html_head(doc: yattag.doc.Doc, title: str) -> None:
"""Produces the <head> tag and its contents."""
prefix = config.Config.get_uri_prefix()
with doc.tag("head"):
with doc.tag("title"):
doc.text(_("Where to map?") + title)
doc.stag("meta", charset="UTF-8")
doc.stag("link",
rel="stylesheet",
type="text/css",
href=prefix + "/static/osm.css")
with doc.tag("script", src=prefix + "/static/sorttable.js"):
pass
doc.stag("meta",
name="viewport",
content="width=device-width, initial-scale=1")
if config.Config.has_matomo():
datadir = config.get_abspath("data")
with open(os.path.join(datadir, "matomo.html.template")) as stream:
matomo_url = config.Config.get_matomo_url()
matomo_site_id = config.Config.get_matomo_site_id()
doc.asis(stream.read().replace("@MATOMO_URL@",
matomo_url).replace(
"@MATOMO_SITE_ID@",
matomo_site_id))
def test_empty_day_range(self) -> None:
"""Tests the case when the day range is empty."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_daily_new(src_root, j, day_range=-1)
daily = j["daily"]
self.assertFalse(daily)
def get_osm_housenumbers_query(self) -> str:
"""Produces a query which lists house numbers in relation."""
datadir = config.get_abspath("data")
with open(os.path.join(datadir,
"street-housenumbers-template.txt")) as stream:
return util.process_template(stream.read(),
self.get_config().get_osmrelation())
def test_empty_day_range(self) -> None:
"""Tests the case when the day range is empty."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_monthly_total(src_root, j, month_range=-1)
monthlytotal = j["monthlytotal"]
self.assertFalse(monthlytotal)
def update_stats_count(today: str) -> None:
"""Counts the # of all house numbers as of today."""
statedir = config.get_abspath("workdir/stats")
csv_path = os.path.join(statedir, "%s.csv" % today)
count_path = os.path.join(statedir, "%s.count" % today)
city_count_path = os.path.join(statedir, "%s.citycount" % today)
house_numbers = set()
cities: Dict[str, Set[str]] = {}
first = True
valid_settlements = util.get_valid_settlements()
with open(csv_path, "r") as stream:
for line in stream.readlines():
if first:
# Ignore the oneliner header.
first = False
continue
# postcode, city name, street name, house number, user
cells = line.split("\t")
# Ignore last column, which is the user who touched the object last.
house_numbers.add("\t".join(cells[:4]))
city_key = util.get_city_key(cells[0], cells[1], valid_settlements)
city_value = "\t".join(cells[2:4])
if city_key in cities:
cities[city_key].add(city_value)
else:
cities[city_key] = set([city_value])
write_count_path(count_path, house_numbers)
write_city_count_path(city_count_path, cities)
def is_missing_housenumbers_html_cached(relation: areas.Relation) -> bool:
"""Decides if we have an up to date cache entry or not."""
cache_path = relation.get_files().get_housenumbers_htmlcache_path()
if not os.path.exists(cache_path):
return False
cache_mtime = os.path.getmtime(cache_path)
osm_streets_path = relation.get_files().get_osm_streets_path()
osm_streets_mtime = os.path.getmtime(osm_streets_path)
if osm_streets_mtime > cache_mtime:
return False
osm_housenumbers_path = relation.get_files().get_osm_housenumbers_path()
osm_housenumbers_mtime = os.path.getmtime(osm_housenumbers_path)
if osm_housenumbers_mtime > cache_mtime:
return False
ref_housenumbers_path = relation.get_files().get_ref_housenumbers_path()
ref_housenumbers_mtime = os.path.getmtime(ref_housenumbers_path)
if ref_housenumbers_mtime > cache_mtime:
return False
datadir = config.get_abspath("data")
relation_path = os.path.join(datadir,
"relation-%s.yaml" % relation.get_name())
relation_mtime = os.path.getmtime(relation_path)
if relation_mtime > cache_mtime:
return False
return True
def test_empty_month_range(self) -> None:
"""Tests the case when the month range is empty."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
stats.handle_monthly_new(src_root, j, month_range=-1)
monthly = j["monthly"]
self.assertTrue(monthly)
def test_old_time(self) -> None:
"""Tests the case when the .count file doesn't exist for a date."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('time.strftime', mock_strftime_old):
stats.handle_progress(src_root, j)
progress = j["progress"]
self.assertEqual(progress["date"], "1970-01-01")
def update_stats(overpass: bool) -> None:
"""Performs the update of country-level stats."""
# Fetch house numbers for the whole country.
info("update_stats: start, updating whole-country csv")
query = util.get_content(
config.get_abspath("data/street-housenumbers-hungary.txt")).decode(
"utf-8")
statedir = config.get_abspath("workdir/stats")
os.makedirs(statedir, exist_ok=True)
today = time.strftime("%Y-%m-%d")
csv_path = os.path.join(statedir, "%s.csv" % today)
if overpass:
retry = 0
while should_retry(retry):
if retry > 0:
info("update_stats: try #%s", retry)
retry += 1
try:
overpass_sleep()
response = overpass_query.overpass_query(query)
with open(csv_path, "w") as stream:
stream.write(response)
break
except urllib.error.HTTPError as http_error:
info("update_stats: http error: %s", str(http_error))
update_stats_count(today)
update_stats_topusers(today)
update_stats_refcount(statedir)
# Remove old CSV files as they are created daily and each is around 11M.
current_time = time.time()
for csv in glob.glob(os.path.join(statedir, "*.csv")):
creation_time = os.path.getmtime(csv)
if (current_time - creation_time) // (24 * 3600) >= 7:
os.unlink(csv)
info("update_stats: removed old %s", csv)
info("update_stats: generating json")
json_path = os.path.join(statedir, "stats.json")
with open(json_path, "w") as stream:
stats.generate_json(statedir, stream)
info("update_stats: end")
def test_old_time(self) -> None:
"""Tests the case when the .count file doesn't exist for a date."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('time.strftime', mock_strftime_old):
stats.handle_topusers(src_root, j)
topusers = j["topusers"]
self.assertFalse(topusers)
def set_language(language: str) -> None:
"""Sets the language of the current thread."""
tls = threading.current_thread.__dict__
localedir = config.get_abspath("locale")
tls["translations"] = gettext.translation("osm-gimmisn",
localedir=localedir,
languages=[language],
fallback=True)
tls["language"] = language
def test_happy(self) -> None:
"""Tests the happy path."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('time.strftime', mock_strftime):
stats.handle_topusers(src_root, j)
topusers = j["topusers"]
self.assertEqual(len(topusers), 20)
self.assertEqual(topusers[0], ["user1", "68885"])
def test_happy(self) -> None:
"""Tests the happy path."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('datetime.date', MockDate):
stats.handle_monthly_total(src_root, j)
monthlytotal = j["monthlytotal"]
self.assertEqual(len(monthlytotal), 1)
self.assertEqual(monthlytotal[0], ['2019-05', 203317])
def test_happy(self) -> None:
"""Tests the happy path."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('datetime.date', MockDate):
stats.handle_user_total(src_root, j)
usertotal = j["usertotal"]
self.assertEqual(len(usertotal), 1)
self.assertEqual(usertotal[0], ["2020-04-27", 43])
def test_happy(self) -> None:
"""Tests the happy path."""
refpath = config.get_abspath(os.path.join("refdir", "utcak_20190514.tsv"))
relations = get_relations()
relation_name = "gazdagret"
relation = relations.get_relation(relation_name)
expected = util.get_content(relations.get_workdir(), "streets-reference-gazdagret.lst")
relation.write_ref_streets(refpath)
actual = util.get_content(relations.get_workdir(), "streets-reference-gazdagret.lst")
self.assertEqual(actual, expected)
def test_happy(self) -> None:
"""Tests the happy path."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('time.strftime', mock_strftime):
stats.handle_progress(src_root, j)
progress = j["progress"]
self.assertEqual(progress["date"], "2020-05-10")
# 254651 / 300 * 100
self.assertEqual(progress["percentage"], 84883.67)
def test_one_element_day_range(self) -> None:
"""Tests the case when the day range is of just one element."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('datetime.date', MockDate):
stats.handle_monthly_total(src_root, j, month_range=0)
monthlytotal = j["monthlytotal"]
self.assertEqual(len(monthlytotal), 2)
self.assertEqual(monthlytotal[0], ["2020-04", 253027])
self.assertEqual(monthlytotal[1], ["2020-05", 254651])
def test_happy(self) -> None:
"""Tests the happy path."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('datetime.date', MockDate):
stats.handle_topcities(src_root, j)
topcities = j["topcities"]
self.assertEqual(len(topcities), 2)
self.assertEqual(topcities[0], ("budapest_02", 190))
self.assertEqual(topcities[1], ("budapest_01", 90))
def test_happy(self) -> None:
"""Tests the happy path."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('datetime.date', MockDate):
# From now on, today is 2020-05-10, so this will read 2020-04-26, 2020-04-27, etc
# (till a file is missing.)
stats.handle_daily_new(src_root, j)
daily = j["daily"]
self.assertEqual(len(daily), 1)
self.assertEqual(daily[0], ["2020-04-26", 364])
def __init__(self, workdir: str) -> None:
self.__workdir = workdir
datadir = config.get_abspath("data")
with open(os.path.join(datadir, "yamls.pickle"), "rb") as stream:
self.__yaml_cache: Dict[str, Any] = pickle.load(stream)
self.__dict = self.__yaml_cache["relations.yaml"]
self.__relations: Dict[str, Relation] = {}
self.__activate_all = False
self.__refcounty_names = self.__yaml_cache["refcounty-names.yaml"]
self.__refsettlement_names = self.__yaml_cache[
"refsettlement-names.yaml"]
def test_happy(self) -> None:
"""Tests the happy path."""
mock_overpass_sleep_called = False
def mock_overpass_sleep() -> None:
nonlocal mock_overpass_sleep_called
mock_overpass_sleep_called = True
result_from_overpass = "******"
result_from_overpass += "7677\tOrfű\tDollár utca\t1\tvasony\n"
result_from_overpass += "7677\tOrfű\tDollár utca\t2\tvasony\n"
def mock_urlopen(_url: str, _data: Optional[bytes] = None) -> BinaryIO:
buf = io.BytesIO()
buf.write(result_from_overpass.encode('utf-8'))
buf.seek(0)
return buf
# Create a CSV that is definitely old enough to be removed.
old_path = config.get_abspath("workdir/stats/old.csv")
create_old_file(old_path)
today = time.strftime("%Y-%m-%d")
path = config.get_abspath("workdir/stats/%s.csv" % today)
with unittest.mock.patch("cron.overpass_sleep", mock_overpass_sleep):
with unittest.mock.patch('urllib.request.urlopen', mock_urlopen):
with unittest.mock.patch('datetime.date', MockDate):
cron.update_stats(overpass=True)
actual = util.get_content(path)
self.assertEqual(actual, result_from_overpass)
# Make sure that the old CSV is removed.
self.assertFalse(os.path.exists(old_path))
self.assertTrue(mock_overpass_sleep_called)
with open(config.get_abspath("workdir/stats/ref.count"),
"r") as stream:
num_ref = int(stream.read().strip())
self.assertEqual(num_ref, 300)
def test_happy(self) -> None:
"""Tests the happy path."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('datetime.date', MockDate):
stats.handle_monthly_new(src_root, j)
monthly = j["monthly"]
self.assertEqual(len(monthly), 2)
# 2019-05 start -> end
self.assertEqual(monthly[0], ["2019-05", 3799])
# diff from last month end -> today
self.assertEqual(monthly[1], ["2020-05", 51334])
def __init__(self, workdir: str, name: str, parent_config: Dict[str, Any],
yaml_cache: Dict[str, Any]) -> None:
self.__workdir = workdir
self.__name = name
my_config: Dict[str, Any] = {}
self.__file = RelationFiles(config.get_abspath("data"), workdir, name)
relation_path = "relation-%s.yaml" % name
# Intentionally don't require this cache to be present, it's fine to omit it for simple
# relations.
if relation_path in yaml_cache:
my_config = yaml_cache[relation_path]
self.__config = RelationConfig(parent_config, my_config)
def test_incomplete_last_month(self) -> None:
"""Tests the case when we have no data for the last, incomplete month."""
src_root = config.get_abspath("workdir/stats")
j: Dict[str, Any] = {}
with unittest.mock.patch('datetime.date', MockDate):
# This would be the data for the current state of the last, incomplete month.
hide_path = config.get_abspath("workdir/stats/2020-05-10.count")
real_exists = os.path.exists
def mock_exists(path: str) -> bool:
if path == hide_path:
return False
return real_exists(path)
with unittest.mock.patch('os.path.exists', mock_exists):
stats.handle_monthly_new(src_root, j)
monthly = j["monthly"]
# 1st element: 2019-05 start -> end
# No 2nd element, would be diff from last month end -> today
self.assertEqual(len(monthly), 1)
self.assertEqual(monthly[0], ["2019-05", 3799])
def handle_github_webhook(environ: Dict[str, Any]) -> yattag.doc.Doc:
"""Handles a GitHub style webhook."""
body = urllib.parse.parse_qs(environ["wsgi.input"].read().decode('utf-8'))
payload = body["payload"][0]
root = json.loads(payload)
if root["ref"] == "refs/heads/master":
my_env = os.environ
my_env["PATH"] = "osm-gimmisn-env/bin:" + my_env["PATH"]
subprocess.run(["make", "-C", config.get_abspath(""), "deploy"], check=True, env=my_env)
return util.html_escape("")
def main() -> None:
"""Commandline interface to this module."""
cache: Dict[str, Any] = {}
datadir = config.get_abspath(sys.argv[1])
for yaml_path in glob.glob(os.path.join(datadir, "*.yaml")):
with open(yaml_path) as yaml_stream:
cache_key = os.path.relpath(yaml_path, datadir)
cache[cache_key] = yaml.safe_load(yaml_stream)
cache_path = os.path.join(datadir, "yamls.pickle")
with open(cache_path, "wb") as cache_stream:
pickle.dump(cache, cache_stream)
