def main() -> None:
"""Commandline interface to this module."""
config = configparser.ConfigParser()
config_path = util.get_abspath("wsgi.ini")
config.read(config_path)
datadir = util.get_abspath("data")
workdir = helpers.get_workdir(config)
relations = helpers.Relations(datadir, workdir)
logpath = os.path.join(workdir, "cron.log")
logging.basicConfig(filename=logpath,
level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logging.getLogger().addHandler(logging.StreamHandler())
start = time.time()
# Query inactive relations once a month.
relations.activate_all(time.localtime(start).tm_mday == 1)
try:
our_main(relations, config)
# pylint: disable=broad-except
except Exception:
logging.error("main: unhandled exception: %s", traceback.format_exc())
delta = time.time() - start
logging.info("main: finished in %s",
str(datetime.timedelta(seconds=delta)))
def main() -> None:
"""Commandline interface to this module."""
config = configparser.ConfigParser()
config_path = util.get_abspath("wsgi.ini")
config.read(config_path)
relation_name = sys.argv[1]
reference = config.get('wsgi', 'reference_housenumbers').strip().split(' ')
workdir = util.get_abspath(config.get('wsgi', 'workdir').strip())
relations = areas.Relations(workdir)
relation = relations.get_relation(relation_name)
relation.write_ref_housenumbers(reference)
def main() -> None:
"""Commandline interface to this module."""
config = configparser.ConfigParser()
config_path = util.get_abspath("wsgi.ini")
config.read(config_path)
relation_name = sys.argv[1]
reference = util.get_abspath(config.get('wsgi', 'reference_street').strip())
datadir = util.get_abspath("data")
workdir = util.get_abspath(config.get('wsgi', 'workdir').strip())
relations = helpers.Relations(datadir, workdir)
relation = relations.get_relation(relation_name)
relation.write_ref_streets(reference)
def main() -> None:
"""Commandline interface."""
config = configparser.ConfigParser()
config_path = util.get_abspath("wsgi.ini")
config.read(config_path)
workdir = util.get_abspath(config.get('wsgi', 'workdir').strip())
relation_name = sys.argv[1]
relations = areas.Relations(workdir)
relation = relations.get_relation(relation_name)
only_in_reference, _ = relation.get_missing_streets()
for street in only_in_reference:
print(street)
def get_osm_housenumbers_query(self) -> str:
"""Produces a query which lists house numbers in relation."""
datadir = util.get_abspath("data")
with open(os.path.join(datadir,
"street-housenumbers-template.txt")) as stream:
return util.process_template(stream.read(),
self.get_config().get_osmrelation())
def __init__(self, *args, **kwargs):
webapp2.RequestHandler.__init__(self, *args, **kwargs)
self.__env = jinja2.Environment(
loader=jinja2.FileSystemLoader(util.get_abspath("template")),
extensions=['jinja2.ext.autoescape'],
autoescape=False)
def missing_streets_update(relations: areas.Relations, relation_name: str) -> yattag.doc.Doc:
"""Expected request_uri: e.g. /osm/missing-streets/ujbuda/update-result."""
reference = util.get_abspath(webframe.get_config().get('wsgi', 'reference_street').strip())
relation = relations.get_relation(relation_name)
relation.write_ref_streets(reference)
doc = yattag.doc.Doc()
with doc.tag("div", id="update-success"):
doc.text(_("Update successful."))
return doc
def handle_github_webhook(environ: Dict[str, Any]) -> yattag.Doc:
"""Handles a GitHub style webhook."""
body = urllib.parse.parse_qs(environ["wsgi.input"].read().decode('utf-8'))
payload = body["payload"][0]
root = json.loads(payload)
if root["ref"] == "refs/heads/master":
subprocess.run(["make", "-C", util.get_abspath(""), "deploy-pythonanywhere"], check=True)
return util.html_escape("")
def set_language(language: str) -> None:
"""Sets the language of the current thread."""
tls = threading.current_thread.__dict__
# Import only inside the function, as util imports a function from this module.
import util
localedir = util.get_abspath("locale")
tls["translations"] = gettext.translation("osm-gimmisn",
localedir=localedir,
languages=[language],
fallback=True)
def __init__(self, workdir: str) -> None:
self.__workdir = workdir
datadir = util.get_abspath("data")
with open(os.path.join(datadir, "yamls.pickle"), "rb") as stream:
self.__yaml_cache: Dict[str, Any] = pickle.load(stream)
self.__dict = self.__yaml_cache["relations.yaml"]
self.__relations: Dict[str, Relation] = {}
self.__activate_all = False
self.__refmegye_names = self.__yaml_cache["refmegye-names.yaml"]
self.__reftelepules_names = self.__yaml_cache["reftelepules-names.yaml"]
def main() -> None:
"""Commandline interface."""
config = configparser.ConfigParser()
config_path = util.get_abspath("wsgi.ini")
config.read(config_path)
workdir = util.get_abspath(config.get('wsgi', 'workdir').strip())
relation_name = sys.argv[1]
relations = areas.Relations(workdir)
relation = relations.get_relation(relation_name)
ongoing_streets, _ = relation.get_missing_housenumbers()
for result in ongoing_streets:
# House number, # of only_in_reference items.
ranges = util.get_housenumber_ranges(result[1])
ranges = sorted(ranges, key=util.split_house_number)
print("%s\t%s" % (result[0], len(ranges)))
# only_in_reference items.
print(ranges)
def missing_housenumbers_update(relations: helpers.Relations, relation_name: str) -> yattag.Doc:
"""Expected request_uri: e.g. /osm/missing-housenumbers/ormezo/update-result."""
reference = get_config().get('wsgi', 'reference_housenumbers').strip().split(' ')
reference = [util.get_abspath(i) for i in reference]
relation = relations.get_relation(relation_name)
relation.write_ref_housenumbers(reference)
doc = yattag.Doc()
doc.text(_("Update successful: "))
link = "/osm/missing-housenumbers/" + relation_name + "/view-result"
doc.asis(util.gen_link(link, _("View missing house numbers")).getvalue())
return doc
def __init__(self, workdir: str, name: str, parent_config: Dict[str, Any],
yaml_cache: Dict[str, Any]) -> None:
self.__workdir = workdir
self.__name = name
my_config: Dict[str, Any] = {}
self.__file = RelationFiles(util.get_abspath("data"), workdir, name)
relation_path = "relation-%s.yaml" % name
# Intentionally don't require this cache to be present, it's fine to omit it for simple
# relations.
if relation_path in yaml_cache:
my_config = yaml_cache[relation_path]
self.__config = RelationConfig(parent_config, my_config)
def main() -> None:
"""Commandline interface to this module."""
cache: Dict[str, Any] = {}
datadir = util.get_abspath(sys.argv[1])
for yaml_path in glob.glob(os.path.join(datadir, "*.yaml")):
with open(yaml_path) as yaml_stream:
cache_key = os.path.relpath(yaml_path, datadir)
cache[cache_key] = yaml.load(yaml_stream)
cache_path = os.path.join(datadir, "yamls.pickle")
with open(cache_path, "wb") as cache_stream:
pickle.dump(cache, cache_stream)
def write_ref_streets(self, reference: str) -> None:
"""Gets known streets (not their coordinates) from a reference site, based on relation names
from OSM."""
# Convert relative path to absolute one.
reference = util.get_abspath(reference)
memory_cache = util.build_street_reference_cache(reference)
lst = self.build_ref_streets(memory_cache)
lst = sorted(set(lst))
with self.get_files().get_ref_streets_stream("w") as sock:
for line in lst:
sock.write(line + "\n")
def handle_static(request_uri: str) -> Tuple[str, str]:
"""Handles serving static content."""
tokens = request_uri.split("/")
path = tokens[-1]
if request_uri.endswith(".js"):
content_type = "application/x-javascript"
elif request_uri.endswith(".css"):
content_type = "text/css"
if path.endswith(".js") or path.endswith(".css"):
return util.get_content(util.get_abspath("static"), path), content_type
return "", ""
def write_ref_housenumbers(self, references: List[str]) -> None:
"""
Writes known house numbers (not their coordinates) from a reference, based on street names
from OSM. Uses build_reference_cache() to build an indexed reference, the result will be
used by __get_ref_housenumbers().
"""
# Convert relative paths to absolute ones.
references = [util.get_abspath(reference) for reference in references]
memory_caches = util.build_reference_caches(references)
streets = self.get_osm_streets()
lst: List[str] = []
for street in streets:
for index, memory_cache in enumerate(memory_caches):
suffix = Relation.__get_ref_suffix(index)
lst += self.build_ref_housenumbers(memory_cache, street, suffix)
lst = sorted(set(lst))
with self.get_files().get_ref_housenumbers_stream("w") as sock:
for line in lst:
sock.write(line + "\n")
def get_workdir(config: configparser.ConfigParser) -> str:
"""Gets the directory which is writable."""
return util.get_abspath(config.get('wsgi', 'workdir').strip())
def missing_streets_update(relations: helpers.Relations, relation_name: str) -> yattag.Doc:
"""Expected request_uri: e.g. /osm/missing-streets/ujbuda/update-result."""
reference = util.get_abspath(get_config().get('wsgi', 'reference_street').strip())
relation = relations.get_relation(relation_name)
relation.write_ref_streets(reference)
return util.html_escape(_("Update successful."))
def get_config() -> configparser.ConfigParser:
"""Gets access to information which are specific to this installation."""
config = configparser.ConfigParser()
config_path = util.get_abspath("wsgi.ini")
config.read(config_path)
return config
def get_datadir() -> str:
"""Gets the directory which is tracked (in version control) data."""
return util.get_abspath("data")
def test_happy(self) -> None:
"""Tests the happy path, when the input is relative."""
actual = util.get_abspath("foo")
expected = os.path.join(os.getcwd(), "foo")
self.assertEqual(actual, expected)
