Exemple #1
0
def handle_main(request_uri: str,
                relations: areas.Relations) -> yattag.doc.Doc:
    """Handles the main wsgi page.

    Also handles /osm/filter-for/* which filters for a condition."""
    filter_for, refcounty = setup_main_filter_for(request_uri)

    doc = yattag.doc.Doc()
    doc.asis(webframe.get_toolbar(relations).getvalue())

    doc.asis(handle_main_filters(relations, refcounty).getvalue())
    table = []
    table.append([
        util.html_escape(_("Area")),
        util.html_escape(_("House number coverage")),
        util.html_escape(_("Existing house numbers")),
        util.html_escape(_("Street coverage")),
        util.html_escape(_("Existing streets")),
        util.html_escape(_("Area boundary"))
    ])
    for relation_name in relations.get_names():
        row = handle_main_relation(relations, filter_for, relation_name)
        if row:
            table.append(row)
    doc.asis(util.html_table_from_list(table).getvalue())
    with doc.tag("p"):
        with doc.tag(
                "a",
                href="https://github.com/vmiklos/osm-gimmisn/tree/master/doc"):
            doc.text(_("Add new area"))

    doc.asis(webframe.get_footer().getvalue())
    return doc
Exemple #2
0
def handle_street_housenumbers(relations: helpers.Relations, request_uri: str) -> yattag.Doc:
    """Expected request_uri: e.g. /osm/street-housenumbers/ormezo/view-query."""
    tokens = request_uri.split("/")
    relation_name = tokens[-2]
    action = tokens[-1]

    relation = relations.get_relation(relation_name)
    osmrelation = relation.get_config().get_osmrelation()

    doc = yattag.Doc()
    doc.asis(get_toolbar(relations, "street-housenumbers", relation_name, osmrelation).getvalue())

    if action == "view-query":
        with doc.tag("pre"):
            doc.text(relation.get_osm_housenumbers_query())
    elif action == "view-result":
        with relation.get_files().get_osm_housenumbers_stream(mode="r") as sock:
            table = util.tsv_to_list(sock)
            doc.asis(util.html_table_from_list(table).getvalue())
    elif action == "update-result":
        query = relation.get_osm_housenumbers_query()
        try:
            relation.get_files().write_osm_housenumbers(overpass_query.overpass_query(query))
            doc.text(_("Update successful: "))
            link = "/osm/missing-housenumbers/" + relation_name + "/view-result"
            doc.asis(util.gen_link(link, _("View missing house numbers")).getvalue())
        except urllib.error.HTTPError as http_error:
            doc.asis(util.handle_overpass_error(http_error).getvalue())

    date = get_housenumbers_last_modified(relation)
    doc.asis(get_footer(date).getvalue())
    return doc
Exemple #3
0
def missing_relations_view_result(relations: helpers.Relations, request_uri: str) -> yattag.Doc:
    """Expected request_uri: e.g. /osm/missing-streets/budapest_11/view-result."""
    tokens = request_uri.split("/")
    relation_name = tokens[-2]
    relation = relations.get_relation(relation_name)

    doc = yattag.Doc()
    if not os.path.exists(relation.get_files().get_osm_streets_path()):
        doc.text(_("No existing streets: "))
        with doc.tag("a", href="/osm/streets/" + relation_name + "/update-result"):
            doc.text(_("Call Overpass to create"))
    elif not os.path.exists(relation.get_files().get_ref_streets_path()):
        doc.text(_("No street list: "))
        with doc.tag("a", href="/osm/missing-streets/" + relation_name + "/update-result"):
            doc.text(_("Create from reference"))
    else:
        ret = relation.write_missing_streets()
        todo_count, done_count, percent, streets = ret
        streets.sort(key=locale.strxfrm)
        table = [[util.html_escape(_("Street name"))]]
        for street in streets:
            table.append([util.html_escape(street)])

        with doc.tag("p"):
            doc.text(_("OpenStreetMap is possibly missing the below {0} streets.").format(str(todo_count)))
            doc.text(_(" (existing: {0}, ready: {1}%).").format(str(done_count), str(percent)))

        doc.asis(util.html_table_from_list(table).getvalue())
    return doc
Exemple #4
0
def get_missing_housenumbers_html(ctx: context.Context,
                                  relation: areas.Relation) -> yattag.doc.Doc:
    """Gets the cached HTML of the missing housenumbers for a relation."""
    doc = yattag.doc.Doc()
    if is_missing_housenumbers_html_cached(ctx, relation):
        with relation.get_files().get_housenumbers_htmlcache_stream(
                "rb") as stream:
            doc.asis(util.from_bytes(stream.read()))
        return doc

    ret = relation.write_missing_housenumbers()
    todo_street_count, todo_count, done_count, percent, table = ret

    with doc.tag("p"):
        prefix = ctx.get_ini().get_uri_prefix()
        relation_name = relation.get_name()
        doc.text(
            tr("OpenStreetMap is possibly missing the below {0} house numbers for {1} streets."
               ).format(str(todo_count), str(todo_street_count)))
        doc.text(
            tr(" (existing: {0}, ready: {1}).").format(
                str(done_count), util.format_percent(str(percent))))
        doc.stag("br")
        with doc.tag(
                "a",
                href="https://github.com/vmiklos/osm-gimmisn/tree/master/doc"):
            doc.text(tr("Filter incorrect information"))
        doc.text(".")
        doc.stag("br")
        with doc.tag(
                "a",
                href=prefix +
                "/missing-housenumbers/{}/view-turbo".format(relation_name)):
            doc.text(tr("Overpass turbo query for the below streets"))
        doc.stag("br")
        with doc.tag("a",
                     href=prefix +
                     "/missing-housenumbers/{}/view-result.txt".format(
                         relation_name)):
            doc.text(tr("Plain text format"))
        doc.stag("br")
        with doc.tag("a",
                     href=prefix +
                     "/missing-housenumbers/{}/view-result.chkl".format(
                         relation_name)):
            doc.text(tr("Checklist format"))

    doc.asis(util.html_table_from_list(table).getvalue())
    doc.asis(
        util.invalid_refstreets_to_html(
            relation.get_invalid_refstreets()).getvalue())
    doc.asis(
        util.invalid_filter_keys_to_html(
            relation.get_invalid_filter_keys()).getvalue())

    with relation.get_files().get_housenumbers_htmlcache_stream(
            "wb") as stream:
        stream.write(util.to_bytes(doc.getvalue()))

    return doc
Exemple #5
0
def missing_streets_view_result(relations: areas.Relations,
                                request_uri: str) -> yattag.doc.Doc:
    """Expected request_uri: e.g. /osm/missing-streets/budapest_11/view-result."""
    tokens = request_uri.split("/")
    relation_name = tokens[-2]
    relation = relations.get_relation(relation_name)

    doc = yattag.doc.Doc()
    prefix = config.Config.get_uri_prefix()
    if not os.path.exists(relation.get_files().get_osm_streets_path()):
        doc.asis(
            webframe.handle_no_osm_streets(prefix, relation_name).getvalue())
        return doc

    if not os.path.exists(relation.get_files().get_ref_streets_path()):
        doc.asis(
            webframe.handle_no_ref_streets(prefix, relation_name).getvalue())
        return doc

    ret = relation.write_missing_streets()
    todo_count, done_count, percent, streets = ret
    streets.sort(key=locale.strxfrm)
    table = [[util.html_escape(_("Street name"))]]
    for street in streets:
        table.append([util.html_escape(street)])

    with doc.tag("p"):
        doc.text(
            _("OpenStreetMap is possibly missing the below {0} streets.").
            format(str(todo_count)))
        doc.text(
            _(" (existing: {0}, ready: {1}).").format(
                str(done_count), util.format_percent(str(percent))))
        doc.stag("br")
        with doc.tag("a",
                     href=prefix +
                     "/missing-streets/{}/view-turbo".format(relation_name)):
            doc.text(
                _("Overpass turbo query for streets with questionable names"))
        doc.stag("br")
        with doc.tag("a",
                     href=prefix + "/missing-streets/" + relation_name +
                     "/view-result.txt"):
            doc.text(_("Plain text format"))
        doc.stag("br")
        with doc.tag("a",
                     href=prefix + "/missing-streets/" + relation_name +
                     "/view-result.chkl"):
            doc.text(_("Checklist format"))

    doc.asis(util.html_table_from_list(table).getvalue())
    doc.asis(
        util.invalid_refstreets_to_html(
            areas.get_invalid_refstreets(relation)).getvalue())
    doc.asis(
        util.invalid_filter_keys_to_html(
            areas.get_invalid_filter_keys(relation)).getvalue())
    return doc
Exemple #6
0
def missing_housenumbers_view_res(relations: areas.Relations,
                                  request_uri: str) -> yattag.doc.Doc:
    """Expected request_uri: e.g. /osm/missing-housenumbers/ormezo/view-result."""
    tokens = request_uri.split("/")
    relation_name = tokens[-2]

    doc = yattag.doc.Doc()
    relation = relations.get_relation(relation_name)
    prefix = config.Config.get_uri_prefix()
    if not os.path.exists(relation.get_files().get_osm_streets_path()):
        with doc.tag("div", id="no-osm-streets"):
            doc.text(_("No existing streets: "))
            link = prefix + "/streets/" + relation_name + "/update-result"
            doc.asis(
                util.gen_link(link, _("Call Overpass to create")).getvalue())
    elif not os.path.exists(relation.get_files().get_osm_housenumbers_path()):
        with doc.tag("div", id="no-osm-housenumbers"):
            doc.text(_("No existing house numbers: "))
            link = prefix + "/street-housenumbers/" + relation_name + "/update-result"
            doc.asis(
                util.gen_link(link, _("Call Overpass to create")).getvalue())
    elif not os.path.exists(relation.get_files().get_ref_housenumbers_path()):
        with doc.tag("div", id="no-ref-housenumbers"):
            doc.text(_("No missing house numbers: "))
            link = prefix + "/missing-housenumbers/" + relation_name + "/update-result"
            doc.asis(
                util.gen_link(link, _("Create from reference")).getvalue())
    else:
        ret = relation.write_missing_housenumbers()
        todo_street_count, todo_count, done_count, percent, table = ret

        with doc.tag("p"):
            doc.text(
                _("OpenStreetMap is possibly missing the below {0} house numbers for {1} streets."
                  ).format(str(todo_count), str(todo_street_count)))
            doc.text(
                _(" (existing: {0}, ready: {1}%).").format(
                    str(done_count), str(percent)))
            doc.stag("br")
            with doc.tag(
                    "a",
                    href=
                    "https://github.com/vmiklos/osm-gimmisn/tree/master/doc"):
                doc.text(_("Filter incorrect information"))
            doc.text(".")
            doc.stag("br")
            with doc.tag(
                    "a",
                    href=prefix +
                    "/missing-housenumbers/{}/view-turbo".format(relation_name)
            ):
                doc.text(_("Overpass turbo query for the below streets"))
            doc.text(".")

        doc.asis(util.html_table_from_list(table).getvalue())
    return doc
Exemple #7
0
 def test_happy(self) -> None:
     """Tests the happy path."""
     fro = [[util.html_escape("A1"),
             util.html_escape("B1")],
            [util.html_escape("A2"),
             util.html_escape("B2")]]
     expected = '<table class="sortable">'
     expected += '<tr><th><a href="#">A1</a></th>'
     expected += '<th><a href="#">B1</a></th></tr>'
     expected += '<tr><td>A2</td><td>B2</td></tr></table>'
     ret = util.html_table_from_list(fro).getvalue()
     self.assertEqual(ret, expected)
Exemple #8
0
def handle_street_housenumbers(ctx: context.Context,
                               relations: areas.Relations,
                               request_uri: str) -> yattag.doc.Doc:
    """Expected request_uri: e.g. /osm/street-housenumbers/ormezo/view-query."""
    tokens = request_uri.split("/")
    relation_name = tokens[-2]
    action = tokens[-1]

    relation = relations.get_relation(relation_name)
    osmrelation = relation.get_config().get_osmrelation()

    doc = yattag.doc.Doc()
    doc.asis(
        webframe.get_toolbar(ctx, relations, "street-housenumbers",
                             relation_name, osmrelation).getvalue())

    prefix = ctx.get_ini().get_uri_prefix()
    if action == "view-query":
        with doc.tag("pre"):
            doc.text(relation.get_osm_housenumbers_query())
    elif action == "update-result":
        query = relation.get_osm_housenumbers_query()
        buf, err = overpass_query.overpass_query(ctx, query)
        if err:
            doc.asis(util.handle_overpass_error(ctx, err).getvalue())
        else:
            relation.get_files().write_osm_housenumbers(ctx, buf)
            doc.text(tr("Update successful: "))
            link = prefix + "/missing-housenumbers/" + relation_name + "/view-result"
            doc.asis(
                util.gen_link(link,
                              tr("View missing house numbers")).getvalue())
    else:
        # assume view-result
        if not ctx.get_file_system().path_exists(
                relation.get_files().get_osm_housenumbers_path()):
            with doc.tag("div", id="no-osm-housenumbers"):
                doc.text(tr("No existing house numbers"))
        else:
            with relation.get_files().get_osm_housenumbers_csv_stream(
                    ctx) as sock:
                doc.asis(
                    util.html_table_from_list(
                        util.tsv_to_list(sock)).getvalue())

    date = get_housenumbers_last_modified(relation)
    doc.asis(webframe.get_footer(date).getvalue())
    return doc
Exemple #9
0
def handle_streets(relations: areas.Relations,
                   request_uri: str) -> yattag.doc.Doc:
    """Expected request_uri: e.g. /osm/streets/ormezo/view-query."""
    tokens = request_uri.split("/")
    relation_name = tokens[-2]
    action = tokens[-1]

    relation = relations.get_relation(relation_name)
    osmrelation = relation.get_config().get_osmrelation()

    doc = yattag.doc.Doc()
    doc.asis(
        webframe.get_toolbar(relations, "streets", relation_name,
                             osmrelation).getvalue())

    prefix = config.Config.get_uri_prefix()
    if action == "view-query":
        with doc.tag("pre"):
            doc.text(relation.get_osm_streets_query())
    elif action == "update-result":
        query = relation.get_osm_streets_query()
        try:
            relation.get_files().write_osm_streets(
                overpass_query.overpass_query(query))
            streets = relation.get_config().should_check_missing_streets()
            if streets != "only":
                doc.text(_("Update successful: "))
                link = prefix + "/missing-housenumbers/" + relation_name + "/view-result"
                doc.asis(
                    util.gen_link(link,
                                  _("View missing house numbers")).getvalue())
            else:
                doc.text(_("Update successful."))
        except urllib.error.HTTPError as http_error:
            doc.asis(util.handle_overpass_error(http_error).getvalue())
    else:
        # assume view-result
        with relation.get_files().get_osm_streets_stream("r") as sock:
            table = util.tsv_to_list(sock)
            doc.asis(util.html_table_from_list(table).getvalue())

    doc.asis(
        webframe.get_footer(get_streets_last_modified(relation)).getvalue())
    return doc
Exemple #10
0
def missing_housenumbers_view_res(relations: areas.Relations, request_uri: str) -> yattag.doc.Doc:
    """Expected request_uri: e.g. /osm/missing-housenumbers/ormezo/view-result."""
    tokens = request_uri.split("/")
    relation_name = tokens[-2]

    doc = yattag.doc.Doc()
    relation = relations.get_relation(relation_name)
    prefix = config.Config.get_uri_prefix()
    if not os.path.exists(relation.get_files().get_osm_streets_path()):
        doc.asis(webframe.handle_no_osm_streets(prefix, relation_name).getvalue())
    elif not os.path.exists(relation.get_files().get_osm_housenumbers_path()):
        doc.asis(webframe.handle_no_osm_housenumbers(prefix, relation_name).getvalue())
    elif not os.path.exists(relation.get_files().get_ref_housenumbers_path()):
        doc.asis(webframe.handle_no_ref_housenumbers(prefix, relation_name).getvalue())
    else:
        ret = relation.write_missing_housenumbers()
        todo_street_count, todo_count, done_count, percent, table = ret

        with doc.tag("p"):
            doc.text(_("OpenStreetMap is possibly missing the below {0} house numbers for {1} streets.")
                     .format(str(todo_count), str(todo_street_count)))
            doc.text(_(" (existing: {0}, ready: {1}).").format(str(done_count), util.format_percent(str(percent))))
            doc.stag("br")
            with doc.tag("a", href="https://github.com/vmiklos/osm-gimmisn/tree/master/doc"):
                doc.text(_("Filter incorrect information"))
            doc.text(".")
            doc.stag("br")
            with doc.tag("a", href=prefix + "/missing-housenumbers/{}/view-turbo".format(relation_name)):
                doc.text(_("Overpass turbo query for the below streets"))
            doc.stag("br")
            with doc.tag("a", href=prefix + "/missing-housenumbers/{}/view-result.txt".format(relation_name)):
                doc.text(_("Plain text format"))
            doc.stag("br")
            with doc.tag("a", href=prefix + "/missing-housenumbers/{}/view-result.chkl".format(relation_name)):
                doc.text(_("Checklist format"))

        doc.asis(util.html_table_from_list(table).getvalue())
        doc.asis(util.invalid_refstreets_to_html(areas.get_invalid_refstreets(relation)).getvalue())
        doc.asis(util.invalid_filter_keys_to_html(areas.get_invalid_filter_keys(relation)).getvalue())
    return doc
Exemple #11
0
def get_additional_housenumbers_html(
        ctx: context.Context, relation: areas.Relation) -> yattag.doc.Doc:
    """Gets the cached HTML of the additional housenumbers for a relation."""
    doc = yattag.doc.Doc()
    if is_additional_housenumbers_html_cached(ctx, relation):
        with relation.get_files().get_additional_housenumbers_htmlcache_stream(
                "rb") as stream:
            doc.asis(util.from_bytes(stream.read()))
        return doc

    ret = relation.write_additional_housenumbers()
    todo_street_count, todo_count, table = ret

    with doc.tag("p"):
        doc.text(
            tr("OpenStreetMap additionally has the below {0} house numbers for {1} streets."
               ).format(str(todo_count), str(todo_street_count)))
        doc.stag("br")
        with doc.tag(
                "a",
                href="https://github.com/vmiklos/osm-gimmisn/tree/master/doc"):
            doc.text(tr("Filter incorrect information"))

    doc.asis(util.html_table_from_list(table).getvalue())
    doc.asis(
        util.invalid_refstreets_to_html(
            relation.get_invalid_refstreets()).getvalue())
    doc.asis(
        util.invalid_filter_keys_to_html(
            relation.get_invalid_filter_keys()).getvalue())

    with relation.get_files().get_additional_housenumbers_htmlcache_stream(
            "wb") as stream:
        stream.write(util.to_bytes(doc.getvalue()))

    return doc
Exemple #12
0
def additional_streets_view_result(ctx: context.Context,
                                   relations: areas.Relations,
                                   request_uri: str) -> yattag.doc.Doc:
    """Expected request_uri: e.g. /osm/additional-streets/budapest_11/view-result."""
    tokens = request_uri.split("/")
    relation_name = tokens[-2]
    relation = relations.get_relation(relation_name)

    doc = yattag.doc.Doc()
    prefix = ctx.get_ini().get_uri_prefix()
    if not ctx.get_file_system().path_exists(
            relation.get_files().get_osm_streets_path()):
        doc.asis(
            webframe.handle_no_osm_streets(prefix, relation_name).getvalue())
    elif not ctx.get_file_system().path_exists(
            relation.get_files().get_ref_streets_path()):
        doc.asis(
            webframe.handle_no_ref_streets(prefix, relation_name).getvalue())
    else:
        # Get "only in OSM" streets.
        streets = relation.write_additional_streets()
        count = len(streets)
        lexical_sort_key = util.get_lexical_sort_key()
        streets.sort(
            key=lambda street: lexical_sort_key(street.get_osm_name()))
        table = [[
            util.html_escape(tr("Identifier")),
            util.html_escape(tr("Type")),
            util.html_escape(tr("Source")),
            util.html_escape(tr("Street name"))
        ]]
        for street in streets:
            cell = yattag.doc.Doc()
            href = "https://www.openstreetmap.org/{}/{}".format(
                street.get_osm_type(), street.get_osm_id())
            with cell.tag("a", href=href, target="_blank"):
                cell.text(str(street.get_osm_id()))
            cells = [
                cell,
                util.html_escape(street.get_osm_type()),
                util.html_escape(street.get_source()),
                util.html_escape(street.get_osm_name()),
            ]
            table.append(cells)

        with doc.tag("p"):
            doc.text(
                tr("OpenStreetMap additionally has the below {0} streets.").
                format(str(count)))
            doc.stag("br")
            with doc.tag("a",
                         href=prefix + "/additional-streets/" + relation_name +
                         "/view-result.txt"):
                doc.text(tr("Plain text format"))
            doc.stag("br")
            with doc.tag("a",
                         href=prefix + "/additional-streets/" + relation_name +
                         "/view-result.chkl"):
                doc.text(tr("Checklist format"))
            doc.stag("br")
            with doc.tag(
                    "a",
                    href=prefix +
                    "/additional-streets/{}/view-turbo".format(relation_name)):
                doc.text(tr("Overpass turbo query for the below streets"))

        doc.asis(util.html_table_from_list(table).getvalue())
        doc.asis(
            util.invalid_refstreets_to_html(
                relation.get_invalid_refstreets()).getvalue())
    return doc
Exemple #13
0
def handle_stats_cityprogress(relations: areas.Relations) -> yattag.doc.Doc:
    """Expected request_uri: e.g. /osm/housenumber-stats/hungary/cityprogress."""
    doc = yattag.doc.Doc()
    doc.asis(get_toolbar(relations).getvalue())

    ref_citycounts: Dict[str, int] = {}
    with open(config.Config.get_reference_citycounts_path(), "r") as stream:
        first = True
        for line in stream.readlines():
            if first:
                first = False
                continue
            cells = line.strip().split('\t')
            if len(cells) < 2:
                continue
            city = cells[0]
            count = int(cells[1])
            ref_citycounts[city] = count
    today = time.strftime("%Y-%m-%d")
    osm_citycounts: Dict[str, int] = {}
    with open(config.Config.get_workdir() + "/stats/" + today + ".citycount",
              "r") as stream:
        for line in stream.readlines():
            cells = line.strip().split('\t')
            if len(cells) < 2:
                continue
            city = cells[0]
            count = int(cells[1])
            osm_citycounts[city] = count
    cities = util.get_in_both(list(ref_citycounts.keys()),
                              list(osm_citycounts.keys()))
    cities.sort(key=locale.strxfrm)
    table = []
    table.append([
        util.html_escape(_("City name")),
        util.html_escape(_("House number coverage")),
        util.html_escape(_("OSM count")),
        util.html_escape(_("Reference count"))
    ])
    for city in cities:
        percent = "100.00"
        if ref_citycounts[city] > 0 and osm_citycounts[city] < ref_citycounts[
                city]:
            percent = "%.2f" % (osm_citycounts[city] / ref_citycounts[city] *
                                100)
        table.append([
            util.html_escape(city),
            util.html_escape(util.format_percent(percent)),
            util.html_escape(str(osm_citycounts[city])),
            util.html_escape(str(ref_citycounts[city]))
        ])
    doc.asis(util.html_table_from_list(table).getvalue())

    with doc.tag("h2"):
        doc.text(_("Note"))
    with doc.tag("div"):
        doc.text(
            _("""These statistics are estimates, not taking house number filters into account.
Only cities with house numbers in OSM are considered."""))

    doc.asis(get_footer().getvalue())
    return doc