def handle_static( request_uri: str) -> Tuple[bytes, str, List[Tuple[str, str]]]: """Handles serving static content.""" tokens = request_uri.split("/") path = tokens[-1] extra_headers: List[Tuple[str, str]] = [] if request_uri.endswith(".js"): content_type = "application/x-javascript" content = util.get_content(config.Config.get_workdir(), path, extra_headers) return content, content_type, extra_headers if request_uri.endswith(".css"): content_type = "text/css" content = util.get_content(config.get_abspath("static"), path, extra_headers) return content, content_type, extra_headers if request_uri.endswith(".json"): content_type = "application/json" content = util.get_content( os.path.join(config.Config.get_workdir(), "stats"), path, extra_headers) return content, content_type, extra_headers if request_uri.endswith(".ico"): content_type = "image/x-icon" content = util.get_content(config.get_abspath(""), path, extra_headers) return content, content_type, extra_headers return bytes(), "", extra_headers
def update_stats() -> None: """Performs the update of country-level stats.""" # Fetch house numbers for the whole country. logging.info("update_stats: start, updating whole-country csv") query = util.get_content( config.get_abspath("data/street-housenumbers-hungary.txt")) statedir = config.get_abspath("workdir/stats") os.makedirs(statedir, exist_ok=True) today = time.strftime("%Y-%m-%d") csv_path = os.path.join(statedir, "%s.csv" % today) retry = 0 while should_retry(retry): if retry > 0: logging.info("update_stats: try #%s", retry) retry += 1 try: overpass_sleep() response = overpass_query.overpass_query(query) with open(csv_path, "w") as stream: stream.write(response) break except urllib.error.HTTPError as http_error: logging.info("update_stats: http error: %s", str(http_error)) # Shell part. logging.info("update_stats: executing the shell part") subprocess.run([config.get_abspath("stats-daily.sh")], check=True) logging.info("update_stats: end")
def test_happy(self) -> None: """Tests the happy path.""" expected = util.get_content(config.get_abspath("workdir/street-housenumbers-reference-gazdagret.lst")) argv = ["", "gazdagret"] with unittest.mock.patch('sys.argv', argv): get_reference_housenumbers.main() actual = util.get_content(config.get_abspath("workdir/street-housenumbers-reference-gazdagret.lst")) self.assertEqual(actual, expected)
def update_stats_count(today: str) -> None: """Counts the # of all house numbers as of today.""" statedir = config.get_abspath("workdir/stats") csv_path = os.path.join(statedir, "%s.csv" % today) count_path = os.path.join(statedir, "%s.count" % today) city_count_path = os.path.join(statedir, "%s.citycount" % today) house_numbers = set() cities: Dict[str, int] = {} first = True with open(csv_path, "r") as stream: for line in stream.readlines(): if first: # Ignore the oneliner header. first = False continue cells = line.split("\t") # Ignore last column, which is the user who touched the object last. house_numbers.add("\t".join(cells[:4])) city_key = util.get_city_key(cells[0], cells[1]) if city_key in cities: cities[city_key] += 1 else: cities[city_key] = 1 with open(count_path, "w") as stream: house_numbers_len = str(len(house_numbers)) stream.write(house_numbers_len + "\n") with open(city_count_path, "w") as stream: for key, value in cities.items(): stream.write(key + "\t" + str(value) + "\n")
def write_html_head(doc: yattag.doc.Doc, title: str) -> None: """Produces the <head> tag and its contents.""" prefix = config.Config.get_uri_prefix() with doc.tag("head"): with doc.tag("title"): doc.text(_("Where to map?") + title) doc.stag("meta", charset="UTF-8") doc.stag("link", rel="stylesheet", type="text/css", href=prefix + "/static/osm.css") with doc.tag("script", src=prefix + "/static/sorttable.js"): pass doc.stag("meta", name="viewport", content="width=device-width, initial-scale=1") if config.Config.has_matomo(): datadir = config.get_abspath("data") with open(os.path.join(datadir, "matomo.html.template")) as stream: matomo_url = config.Config.get_matomo_url() matomo_site_id = config.Config.get_matomo_site_id() doc.asis(stream.read().replace("@MATOMO_URL@", matomo_url).replace( "@MATOMO_SITE_ID@", matomo_site_id))
def test_empty_day_range(self) -> None: """Tests the case when the day range is empty.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_daily_new(src_root, j, day_range=-1) daily = j["daily"] self.assertFalse(daily)
def get_osm_housenumbers_query(self) -> str: """Produces a query which lists house numbers in relation.""" datadir = config.get_abspath("data") with open(os.path.join(datadir, "street-housenumbers-template.txt")) as stream: return util.process_template(stream.read(), self.get_config().get_osmrelation())
def test_empty_day_range(self) -> None: """Tests the case when the day range is empty.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_monthly_total(src_root, j, month_range=-1) monthlytotal = j["monthlytotal"] self.assertFalse(monthlytotal)
def update_stats_count(today: str) -> None: """Counts the # of all house numbers as of today.""" statedir = config.get_abspath("workdir/stats") csv_path = os.path.join(statedir, "%s.csv" % today) count_path = os.path.join(statedir, "%s.count" % today) city_count_path = os.path.join(statedir, "%s.citycount" % today) house_numbers = set() cities: Dict[str, Set[str]] = {} first = True valid_settlements = util.get_valid_settlements() with open(csv_path, "r") as stream: for line in stream.readlines(): if first: # Ignore the oneliner header. first = False continue # postcode, city name, street name, house number, user cells = line.split("\t") # Ignore last column, which is the user who touched the object last. house_numbers.add("\t".join(cells[:4])) city_key = util.get_city_key(cells[0], cells[1], valid_settlements) city_value = "\t".join(cells[2:4]) if city_key in cities: cities[city_key].add(city_value) else: cities[city_key] = set([city_value]) write_count_path(count_path, house_numbers) write_city_count_path(city_count_path, cities)
def is_missing_housenumbers_html_cached(relation: areas.Relation) -> bool: """Decides if we have an up to date cache entry or not.""" cache_path = relation.get_files().get_housenumbers_htmlcache_path() if not os.path.exists(cache_path): return False cache_mtime = os.path.getmtime(cache_path) osm_streets_path = relation.get_files().get_osm_streets_path() osm_streets_mtime = os.path.getmtime(osm_streets_path) if osm_streets_mtime > cache_mtime: return False osm_housenumbers_path = relation.get_files().get_osm_housenumbers_path() osm_housenumbers_mtime = os.path.getmtime(osm_housenumbers_path) if osm_housenumbers_mtime > cache_mtime: return False ref_housenumbers_path = relation.get_files().get_ref_housenumbers_path() ref_housenumbers_mtime = os.path.getmtime(ref_housenumbers_path) if ref_housenumbers_mtime > cache_mtime: return False datadir = config.get_abspath("data") relation_path = os.path.join(datadir, "relation-%s.yaml" % relation.get_name()) relation_mtime = os.path.getmtime(relation_path) if relation_mtime > cache_mtime: return False return True
def test_empty_month_range(self) -> None: """Tests the case when the month range is empty.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} stats.handle_monthly_new(src_root, j, month_range=-1) monthly = j["monthly"] self.assertTrue(monthly)
def test_old_time(self) -> None: """Tests the case when the .count file doesn't exist for a date.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('time.strftime', mock_strftime_old): stats.handle_progress(src_root, j) progress = j["progress"] self.assertEqual(progress["date"], "1970-01-01")
def update_stats(overpass: bool) -> None: """Performs the update of country-level stats.""" # Fetch house numbers for the whole country. info("update_stats: start, updating whole-country csv") query = util.get_content( config.get_abspath("data/street-housenumbers-hungary.txt")).decode( "utf-8") statedir = config.get_abspath("workdir/stats") os.makedirs(statedir, exist_ok=True) today = time.strftime("%Y-%m-%d") csv_path = os.path.join(statedir, "%s.csv" % today) if overpass: retry = 0 while should_retry(retry): if retry > 0: info("update_stats: try #%s", retry) retry += 1 try: overpass_sleep() response = overpass_query.overpass_query(query) with open(csv_path, "w") as stream: stream.write(response) break except urllib.error.HTTPError as http_error: info("update_stats: http error: %s", str(http_error)) update_stats_count(today) update_stats_topusers(today) update_stats_refcount(statedir) # Remove old CSV files as they are created daily and each is around 11M. current_time = time.time() for csv in glob.glob(os.path.join(statedir, "*.csv")): creation_time = os.path.getmtime(csv) if (current_time - creation_time) // (24 * 3600) >= 7: os.unlink(csv) info("update_stats: removed old %s", csv) info("update_stats: generating json") json_path = os.path.join(statedir, "stats.json") with open(json_path, "w") as stream: stats.generate_json(statedir, stream) info("update_stats: end")
def test_old_time(self) -> None: """Tests the case when the .count file doesn't exist for a date.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('time.strftime', mock_strftime_old): stats.handle_topusers(src_root, j) topusers = j["topusers"] self.assertFalse(topusers)
def set_language(language: str) -> None: """Sets the language of the current thread.""" tls = threading.current_thread.__dict__ localedir = config.get_abspath("locale") tls["translations"] = gettext.translation("osm-gimmisn", localedir=localedir, languages=[language], fallback=True) tls["language"] = language
def test_happy(self) -> None: """Tests the happy path.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('time.strftime', mock_strftime): stats.handle_topusers(src_root, j) topusers = j["topusers"] self.assertEqual(len(topusers), 20) self.assertEqual(topusers[0], ["user1", "68885"])
def test_happy(self) -> None: """Tests the happy path.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('datetime.date', MockDate): stats.handle_monthly_total(src_root, j) monthlytotal = j["monthlytotal"] self.assertEqual(len(monthlytotal), 1) self.assertEqual(monthlytotal[0], ['2019-05', 203317])
def test_happy(self) -> None: """Tests the happy path.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('datetime.date', MockDate): stats.handle_user_total(src_root, j) usertotal = j["usertotal"] self.assertEqual(len(usertotal), 1) self.assertEqual(usertotal[0], ["2020-04-27", 43])
def test_happy(self) -> None: """Tests the happy path.""" refpath = config.get_abspath(os.path.join("refdir", "utcak_20190514.tsv")) relations = get_relations() relation_name = "gazdagret" relation = relations.get_relation(relation_name) expected = util.get_content(relations.get_workdir(), "streets-reference-gazdagret.lst") relation.write_ref_streets(refpath) actual = util.get_content(relations.get_workdir(), "streets-reference-gazdagret.lst") self.assertEqual(actual, expected)
def test_happy(self) -> None: """Tests the happy path.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('time.strftime', mock_strftime): stats.handle_progress(src_root, j) progress = j["progress"] self.assertEqual(progress["date"], "2020-05-10") # 254651 / 300 * 100 self.assertEqual(progress["percentage"], 84883.67)
def test_one_element_day_range(self) -> None: """Tests the case when the day range is of just one element.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('datetime.date', MockDate): stats.handle_monthly_total(src_root, j, month_range=0) monthlytotal = j["monthlytotal"] self.assertEqual(len(monthlytotal), 2) self.assertEqual(monthlytotal[0], ["2020-04", 253027]) self.assertEqual(monthlytotal[1], ["2020-05", 254651])
def test_happy(self) -> None: """Tests the happy path.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('datetime.date', MockDate): stats.handle_topcities(src_root, j) topcities = j["topcities"] self.assertEqual(len(topcities), 2) self.assertEqual(topcities[0], ("budapest_02", 190)) self.assertEqual(topcities[1], ("budapest_01", 90))
def test_happy(self) -> None: """Tests the happy path.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('datetime.date', MockDate): # From now on, today is 2020-05-10, so this will read 2020-04-26, 2020-04-27, etc # (till a file is missing.) stats.handle_daily_new(src_root, j) daily = j["daily"] self.assertEqual(len(daily), 1) self.assertEqual(daily[0], ["2020-04-26", 364])
def __init__(self, workdir: str) -> None: self.__workdir = workdir datadir = config.get_abspath("data") with open(os.path.join(datadir, "yamls.pickle"), "rb") as stream: self.__yaml_cache: Dict[str, Any] = pickle.load(stream) self.__dict = self.__yaml_cache["relations.yaml"] self.__relations: Dict[str, Relation] = {} self.__activate_all = False self.__refcounty_names = self.__yaml_cache["refcounty-names.yaml"] self.__refsettlement_names = self.__yaml_cache[ "refsettlement-names.yaml"]
def test_happy(self) -> None: """Tests the happy path.""" mock_overpass_sleep_called = False def mock_overpass_sleep() -> None: nonlocal mock_overpass_sleep_called mock_overpass_sleep_called = True result_from_overpass = "******" result_from_overpass += "7677\tOrfű\tDollár utca\t1\tvasony\n" result_from_overpass += "7677\tOrfű\tDollár utca\t2\tvasony\n" def mock_urlopen(_url: str, _data: Optional[bytes] = None) -> BinaryIO: buf = io.BytesIO() buf.write(result_from_overpass.encode('utf-8')) buf.seek(0) return buf # Create a CSV that is definitely old enough to be removed. old_path = config.get_abspath("workdir/stats/old.csv") create_old_file(old_path) today = time.strftime("%Y-%m-%d") path = config.get_abspath("workdir/stats/%s.csv" % today) with unittest.mock.patch("cron.overpass_sleep", mock_overpass_sleep): with unittest.mock.patch('urllib.request.urlopen', mock_urlopen): with unittest.mock.patch('datetime.date', MockDate): cron.update_stats(overpass=True) actual = util.get_content(path) self.assertEqual(actual, result_from_overpass) # Make sure that the old CSV is removed. self.assertFalse(os.path.exists(old_path)) self.assertTrue(mock_overpass_sleep_called) with open(config.get_abspath("workdir/stats/ref.count"), "r") as stream: num_ref = int(stream.read().strip()) self.assertEqual(num_ref, 300)
def test_happy(self) -> None: """Tests the happy path.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('datetime.date', MockDate): stats.handle_monthly_new(src_root, j) monthly = j["monthly"] self.assertEqual(len(monthly), 2) # 2019-05 start -> end self.assertEqual(monthly[0], ["2019-05", 3799]) # diff from last month end -> today self.assertEqual(monthly[1], ["2020-05", 51334])
def __init__(self, workdir: str, name: str, parent_config: Dict[str, Any], yaml_cache: Dict[str, Any]) -> None: self.__workdir = workdir self.__name = name my_config: Dict[str, Any] = {} self.__file = RelationFiles(config.get_abspath("data"), workdir, name) relation_path = "relation-%s.yaml" % name # Intentionally don't require this cache to be present, it's fine to omit it for simple # relations. if relation_path in yaml_cache: my_config = yaml_cache[relation_path] self.__config = RelationConfig(parent_config, my_config)
def test_incomplete_last_month(self) -> None: """Tests the case when we have no data for the last, incomplete month.""" src_root = config.get_abspath("workdir/stats") j: Dict[str, Any] = {} with unittest.mock.patch('datetime.date', MockDate): # This would be the data for the current state of the last, incomplete month. hide_path = config.get_abspath("workdir/stats/2020-05-10.count") real_exists = os.path.exists def mock_exists(path: str) -> bool: if path == hide_path: return False return real_exists(path) with unittest.mock.patch('os.path.exists', mock_exists): stats.handle_monthly_new(src_root, j) monthly = j["monthly"] # 1st element: 2019-05 start -> end # No 2nd element, would be diff from last month end -> today self.assertEqual(len(monthly), 1) self.assertEqual(monthly[0], ["2019-05", 3799])
def handle_github_webhook(environ: Dict[str, Any]) -> yattag.doc.Doc: """Handles a GitHub style webhook.""" body = urllib.parse.parse_qs(environ["wsgi.input"].read().decode('utf-8')) payload = body["payload"][0] root = json.loads(payload) if root["ref"] == "refs/heads/master": my_env = os.environ my_env["PATH"] = "osm-gimmisn-env/bin:" + my_env["PATH"] subprocess.run(["make", "-C", config.get_abspath(""), "deploy"], check=True, env=my_env) return util.html_escape("")
def main() -> None: """Commandline interface to this module.""" cache: Dict[str, Any] = {} datadir = config.get_abspath(sys.argv[1]) for yaml_path in glob.glob(os.path.join(datadir, "*.yaml")): with open(yaml_path) as yaml_stream: cache_key = os.path.relpath(yaml_path, datadir) cache[cache_key] = yaml.safe_load(yaml_stream) cache_path = os.path.join(datadir, "yamls.pickle") with open(cache_path, "wb") as cache_stream: pickle.dump(cache, cache_stream)