def errors(db, lang): params = query._params(max_limit=10000) results = query._gets(db, params) translate = utils.translator(lang) out = [] for res in results: i = { 'lat': float(res["lat"]), 'lon': float(res["lon"]), 'id': res["id"], 'item': str(res["item"]), } if params.full: i.update({ 'lat': float(res["lat"]), 'lon': float(res["lon"]), 'id': res["id"], 'item': str(res["item"]), 'source': res["source"], 'classs': res["class"], 'elems': res["elems"], 'subclass': res["subclass"], 'subtitle': translate.select(res["subtitle"]), 'title': translate.select(res["title"]), 'level': res["level"], 'update': str(res["timestamp"]), 'username': res["username"] or None, }) out.append(i) return {'issues': out}
def updates(db, lang): db.execute(""" SELECT source.id, EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp)) AS age, source.country, source.analyser FROM source LEFT JOIN dynpoi_update_last ON source.id = dynpoi_update_last.source ORDER BY dynpoi_update_last.timestamp DESC """) liste = [] for res in db.fetchall(): (source, age, country, analyser) = (res[0], res[1], res[2], res[3]) if age: if age >= 0: # TRANSLATORS: days / hours / minutes since last source update, abbreviated to d / h / m txt = _("{day}d, {hour}h, {minute}m ago").format(day=int(age/86400), hour=int(age/3600)%24, minute=int(age/60)%60) else: txt = _("in {day}d, {hour}h, {minute}m").format(day=int(-age/86400), hour=int(-age/3600)%24, minute=int(-age/60)%60) liste.append((country, analyser, age, txt, source)) else: liste.append((country, analyser, 1e10, _("never generated"), source)) liste.sort(lambda x, y: -cmp(x[2], y[2])) return template('control/updates', translate=utils.translator(lang), liste=liste)
def updates(db, lang): db.execute(""" SELECT analyser, COUNT(*), MIN(EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp)))/60/60/24 AS min_age, MAX(EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp)))/60/60/24 AS max_age, MIN(dynpoi_update_last.version) AS min_version, MAX(dynpoi_update_last.version) AS max_version FROM source JOIN dynpoi_update_last ON source.id = dynpoi_update_last.source WHERE dynpoi_update_last.version IS NOT NULL AND dynpoi_update_last.version NOT IN ('(None)', '(unknown)') GROUP BY analyser ORDER BY analyser """) summary = defaultdict(list) max_versions = None for res in db.fetchall(): (analyser, count, min_age, max_age, min_version, max_version) = res max_versions = max_version if max_version > max_versions else max_versions summary[analyser] = {'count': count, 'min_age': min_age, 'max_age': max_age, 'max_version': '-'.join((max_version or '').split('-')[1:5]), 'min_version': '-'.join((min_version or '').split('-')[1:5])} max_versions = '-'.join((max_versions or '').split('-')[1:5]) return template('control/updates_summary_by_analyser', translate=utils.translator(lang), summary=summary, max_versions=max_versions)
def _errors(db, lang, params): results = query._gets(db, params) out = OrderedDict() if not params.full: out["description"] = ["lat", "lon", "error_id", "item"] else: out["description"] = ["lat", "lon", "error_id", "item", "source", "class", "elems", "subclass", "subtitle", "title", "level", "update", "username"] out["errors"] = [] translate = utils.translator(lang) for res in results: lat = res["lat"] lon = res["lon"] error_id = res["id"] item = res["item"] or 0 if not params.full: out["errors"].append([str(lat), str(lon), str(error_id), str(item)]) else: source = res["source"] classs = res["class"] elems = res["elems"] subclass = res["subclass"] subtitle = translate.select(res["subtitle"]) title = translate.select(res["title"]) level = res["level"] update = res["timestamp"] username = res["username"] or "" out["errors"].append([str(lat), str(lon), str(error_id), str(item), str(source), str(classs), str(elems), str(subclass), subtitle, title, str(level), str(update), username]) return out
def fp(db, lang, err_id): data_type = { "N": "node", "W": "way", "R": "relation", "I": "infos"} # TRANSLATORS: link to tooltip help url_help = _("http://wiki.openstreetmap.org/wiki/Osmose/errors") translate = utils.translator(lang) (marker, columns) = _get(db, err_id, 'false') lat = str(marker["lat"]) lon = str(marker["lon"]) title = translate.select(marker["title"]) subtitle = translate.select(marker["subtitle"]) b_date = marker["timestamp"] or "" item = marker["item"] or 0 date = marker["date"].isoformat() or 0 return { "lat":lat, "lon":lon, "minlat": float(lat) - 0.002, "maxlat": float(lat) + 0.002, "minlon": float(lon) - 0.002, "maxlon": float(lon) + 0.002, "error_id":err_id, "title":title, "subtitle":subtitle, "b_date":b_date.strftime("%Y-%m-%d"), "item":item, "date":date, "url_help":url_help }
def fp(db, lang, err_id): data_type = {"N": "node", "W": "way", "R": "relation", "I": "infos"} # TRANSLATORS: link to tooltip help url_help = _("http://wiki.openstreetmap.org/wiki/Osmose/errors") translate = utils.translator(lang) (marker, columns) = _get(db, err_id, 'false') lat = str(marker["lat"]) lon = str(marker["lon"]) title = translate.select(marker["title"]) subtitle = translate.select(marker["subtitle"]) b_date = marker["timestamp"] or "" item = marker["item"] or 0 date = marker["date"].isoformat() or 0 return { "lat": lat, "lon": lon, "minlat": float(lat) - 0.002, "maxlat": float(lat) + 0.002, "minlon": float(lon) - 0.002, "maxlon": float(lon) + 0.002, "error_id": err_id, "title": title, "subtitle": subtitle, "b_date": b_date.strftime("%Y-%m-%d"), "item": item, "date": date, "url_help": url_help }
def user(db, lang, username=None, format=None): params = query._params() if username: params.users = utils.pg_escape(username.decode("utf-8")).split(",") params.limit = 500 params.full = True username = "******".join(params.users) if not params.users: return template('byuser/index', translate=utils.translator(lang)) errors = query._gets(db, params) count = len(errors) if request.path.startswith("/api") or format == "json": out = OrderedDict() for res in errors: res["timestamp"] = str(res["timestamp"]) res["lat"] = float(res["lat"]) res["lon"] = float(res["lon"]) out["issues"] = map(dict, errors) return out elif format == 'rss': response.content_type = "application/rss+xml" return template('byuser/byuser.rss', username=username, users=params.users, count=count, errors=errors, translate=utils.translator(lang), website=utils.website) else: return template('byuser/byuser', username=username, users=params.users, count=count, errors=errors, translate=utils.translator(lang), website=utils.website, main_website=utils.main_website, remote_url_read=utils.remote_url_read, html_escape=html_escape)
def user(db, lang, username=None, format=None): params = query._params() if username: params.users = utils.pg_escape(username.decode("utf-8")).split(",") params.limit = 500 params.full = True username = "******".join(params.users) if not params.users: return template('byuser/index', translate=utils.translator(lang)) errors = query._gets(db, params) count = len(errors) if request.path.startswith("/api") or format == "json": out = OrderedDict() out["description"] = ["id", "item", "lat", "lon", "source", "class", "elems", "subclass", "subtitle", "comment", "title", "level", "timestamp", "menu", "username", "date"] for res in errors: res["timestamp"] = str(res["timestamp"]) res["date"] = str(res["date"]) out["byusers"] = errors return out elif format == 'rss': response.content_type = "application/rss+xml" return template('byuser/byuser.rss', username=username, users=params.users, count=count, errors=errors, translate=utils.translator(lang), website=utils.website) else: return template('byuser/byuser', username=username, users=params.users, count=count, errors=errors, translate=utils.translator(lang), website=utils.website, main_website=utils.main_website, remote_url_read=utils.remote_url_read, html_escape=html_escape)
def user(db, lang, username=None, format=None): params = query._params() if username: params.users = utils.pg_escape(username.decode("utf-8")).split(",") params.limit = 500 params.full = True username = "******".join(params.users) if not params.users: return template('byuser/index') errors = query._gets(db, params) count = len(errors) if request.path.startswith("/api") or format == "json": out = OrderedDict() out["description"] = [ "id", "item", "lat", "lon", "source", "class", "elems", "subclass", "subtitle", "comment", "title", "level", "timestamp", "menu", "username", "date" ] for res in errors: res["timestamp"] = str(res["timestamp"]) res["date"] = str(res["date"]) out["byusers"] = errors return out elif format == 'rss': response.content_type = "application/rss+xml" return template('byuser/byuser.rss', username=username, users=params.users, count=count, errors=errors, translate=utils.translator(lang), website=utils.website) else: return template('byuser/byuser', username=username, users=params.users, count=count, errors=errors, translate=utils.translator(lang), website=utils.website, main_website=utils.main_website, remote_url_read=utils.remote_url_read, html_escape=html_escape)
def fp_(db, lang, err_id): (marker, columns) = _get(db, err_id, 'false') return template('false-positive/index', translate=utils.translator(lang), err_id=err_id, marker=marker, columns_marker=columns)
def updates(db, lang): remote = request.params.get('remote') country = request.params.get('country') db.execute(""" SELECT DISTINCT ON (source.id) source.id, EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp)) AS age, country, analyser FROM source JOIN dynpoi_update_last ON source.id = dynpoi_update_last.source WHERE """ + (""" RIGHT(MD5(remote_ip), 4) = %(remote)s AND """ if remote else "") + (""" source.country LIKE %(country)s AND """ if country else "") + """ true ORDER BY source.id ASC, dynpoi_update_last.timestamp DESC """, {"remote": remote, "country": country and country.replace("*", "%")}) keys = defaultdict(int) matrix = defaultdict(dict) stats_analyser = {} stats_country = {} for res in db.fetchall(): (source, age, country, analyser) = (res[0], res[1], res[2], res[3]) keys[country] += 1 matrix[analyser][country] = (age/60/60/24, source) for analyser in matrix: min = max = None sum = 0 for country in matrix[analyser]: v = matrix[analyser][country][0] min = v if not min or v < min else min max = v if not max or v > max else max sum += v if not stats_country.has_key(country): min_c = v sum_c = v max_c = v n_c = 1 else: (min_c, sum_c, max_c, n_c) = stats_country[country] min_c = v if v < min_c else min_c max_c = v if v > max_c else max_c sum_c += v n_c += 1 stats_country[country] = [min_c, sum_c, max_c, n_c] stats_analyser[analyser] = [min, sum/len(matrix[analyser]), max] avg_country = {} for country in stats_country: stats_country[country][1] = stats_country[country][1]/stats_country[country][3] keys = sorted(keys.keys()) return template('control/updates_matrix', translate=utils.translator(lang), keys=keys, matrix=matrix, stats_analyser=stats_analyser, stats_country=stats_country)
def getBugsByUser(db, lang, user=None): user = user or request.params.get('user') clause, order = bboxRequest2Clause() sql_arg = {} limit = '' if user: clause.append("u.username = %(username)s") sql_arg['username'] = user else: num_points = request.params.get('points', type=int, default=100) if num_points != "all": limit = "LIMIT %d" % int(num_points) list2Clause(request.params.get('class'), clause, 'm.class') list2Clause(request.params.item, clause, 'm.item') list2Clause(request.params.not_item, clause, 'm.item NOT') sqlbase = """ SELECT marker.id, marker.item, marker.source, marker.class, marker.elems, marker.subclass, marker.lat, marker.lon, dynpoi_class.title as title, marker.subtitle as subtitle, dynpoi_update_last.timestamp, u.username FROM marker INNER JOIN dynpoi_class ON marker.source=dynpoi_class.source AND marker.class=dynpoi_class.class INNER JOIN dynpoi_update_last ON marker.source = dynpoi_update_last.source LEFT JOIN marker_elem u ON marker.id = u.marker_id WHERE %s ORDER BY dynpoi_update_last.timestamp DESC %s """ translate = utils.translator(lang) def each(res): lat = res["lat"] lon = res["lon"] error_id = res["id"] title = translate.select(res["title"]) subtitle = translate.select(res["subtitle"]) item = res["item"] or 0 return u'"%s", "%s", "%s", %f, %f, "%s", "%s", "%s"\n' % (res["timestamp"], res["username"].decode('utf-8'), error_id, lon, lat, title, subtitle, item) response.content_type = 'text/plain; Charset=UTF-8' content = "# timestamp, username, error_id, lon, lat, title, subtitle, item\n" return content + query(db, sqlbase % (' AND '.join(clause), limit), sql_arg, each)
def updates(db, lang): db.execute(""" SELECT coalesce(backend.hostname, dynpoi_update_last.remote_ip) AS remote, RIGHT(MD5(remote_ip), 4) AS remote_ip_hash, country, MAX(EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp))) AS max_age, MIN(EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp))) AS min_age, MAX(dynpoi_update_last.version) AS max_version, MIN(dynpoi_update_last.version) AS min_version, count(*) AS count FROM source JOIN dynpoi_update_last ON source.id = dynpoi_update_last.source LEFT JOIN backend ON dynpoi_update_last.remote_ip = backend.ip GROUP BY remote, remote_ip_hash, country ORDER BY remote, MIN(EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp))) ASC """) summary = defaultdict(list) remote_hashes = {} max_versions = defaultdict(list) min_versions = defaultdict(list) for res in db.fetchall(): (remote, remote_hash, country, max_age, min_age, max_version, min_version, count) = res summary[remote].append({ 'country': country, 'max_age': max_age / 60 / 60 / 24, 'min_age': min_age / 60 / 60 / 24, 'count': count }) remote_hashes[remote] = remote_hash max_versions[remote].append(max_version) min_versions[remote].append(min_version) for remote in max_versions.keys(): max_versions[remote] = max(max_versions[remote]) if max_versions[remote] and '-' in max_versions[remote]: max_versions[remote] = '-'.join( max_versions[remote].split('-')[1:5]) min_versions[remote] = min(min_versions[remote]) if min_versions[remote] and '-' in min_versions[remote]: min_versions[remote] = '-'.join( min_versions[remote].split('-')[1:5]) return template('control/updates_summary', translate=utils.translator(lang), summary=summary, max_versions=max_versions, min_versions=min_versions, remote_hashes=remote_hashes)
def _errors_geo(db, lang, params): results = query._gets(db, params) translate = utils.translator(lang) features = [] for res in results: properties = {"error_id": res["id"], "item": res["item"] or 0} features.append({"type": "Feature", "geometry": {"type": "Point", "coordinates": [float(res["lon"]), float(res["lat"])]}, "properties": properties}) return {"type": "FeatureCollection", "features": features}
def updates(db, lang): db.execute(""" SELECT backend.hostname AS hostname, dynpoi_update_last.remote_ip AS remote, RIGHT(MD5(remote_ip), 4) AS remote_ip_hash, country, MAX(EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp))) AS max_age, MIN(EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp))) AS min_age, MAX(dynpoi_update_last.version) AS max_version, MIN(dynpoi_update_last.version) AS min_version, count(*) AS count FROM source JOIN dynpoi_update_last ON source.id = dynpoi_update_last.source LEFT JOIN backend ON dynpoi_update_last.remote_ip = backend.ip GROUP BY hostname, remote_ip, remote_ip_hash, country ORDER BY min_age ASC """) summary = defaultdict(list) remote_hashes = {} hostnames = defaultdict(list) max_versions = defaultdict(list) min_versions = defaultdict(list) max_count = 0 for res in db.fetchall(): (hostname, remote, remote_hash, country, max_age, min_age, max_version, min_version, count) = res max_count = max(max_count, count) summary[remote].append({'hostname': hostname, 'country': country, 'max_age': max_age/60/60/24, 'min_age': min_age/60/60/24, 'count': count}) remote_hashes[remote] = remote_hash hostnames[remote].append(hostname) max_versions[remote].append(max_version) min_versions[remote].append(min_version) for remote in max_versions.keys(): hostnames[remote] = hostnames[remote][0] max_versions[remote] = max(max_versions[remote]) if max_versions[remote] and '-' in max_versions[remote]: max_versions[remote] = '-'.join(max_versions[remote].split('-')[1:5]) min_versions[remote] = min(min_versions[remote]) if min_versions[remote] and '-' in min_versions[remote]: min_versions[remote] = '-'.join(min_versions[remote].split('-')[1:5]) return template('control/updates_summary', translate=utils.translator(lang), summary=summary, hostnames=hostnames, max_versions=max_versions, min_versions=min_versions, remote_hashes=remote_hashes, max_count=max_count)
def user_count(db, lang, username=None, format=None): count = _user_count(db, username) if request.path.startswith("/api") or format == "json": return count elif format == 'rss': response.content_type = "application/rss+xml" return template('byuser/byuser_count.rss', username=username, count=count, translate=utils.translator(lang), website=utils.website) else: return count
def getBugs(db, lang): clause, order = bboxRequest2Clause() list2Clause(request.params.get('item'), clause, 'marker.item') sqlbase = """ SELECT marker.id, marker.item, marker.lat, marker.lon, dynpoi_class.title as title, marker.subtitle as subtitle FROM marker INNER JOIN dynpoi_class ON marker.source=dynpoi_class.source AND marker.class=dynpoi_class.class INNER JOIN dynpoi_update_last ON marker.source = dynpoi_update_last.source WHERE %s AND dynpoi_update_last.timestamp > (now() - interval '3 months') ORDER BY %s LIMIT 100 """ translate = utils.translator(lang) def each(res): lat = res["lat"] lon = res["lon"] error_id = res["id"] title = translate.select(res["title"]) subtitle = translate.select(res["subtitle"]) item = res["item"] or 0 text = title if subtitle: text += " - <br>" + subtitle return u"putAJAXMarker('%s', %f, %f, '%s', '%s');\n" % ( error_id, lon, lat, text, item) response.content_type = 'text/plain; Charset=UTF-8' return query(db, sqlbase % (' AND '.join(clause), order), None, each)
def _errors_geo(db, lang, params): results = query._gets(db, params) translate = utils.translator(lang) features = [] for res in results: properties = {"error_id": res["id"], "item": res["item"] or 0} features.append({ "type": "Feature", "geometry": { "type": "Point", "coordinates": [float(res["lon"]), float(res["lat"])] }, "properties": properties }) return {"type": "FeatureCollection", "features": features}
def getBugs(db, lang): clause, order = bboxRequest2Clause() list2Clause(request.params.get('item'), clause, 'marker.item') sqlbase = """ SELECT marker.id, marker.item, marker.lat, marker.lon, dynpoi_class.title as title, marker.subtitle as subtitle FROM marker INNER JOIN dynpoi_class ON marker.source=dynpoi_class.source AND marker.class=dynpoi_class.class INNER JOIN dynpoi_update_last ON marker.source = dynpoi_update_last.source WHERE %s AND dynpoi_update_last.timestamp > (now() - interval '3 months') ORDER BY %s LIMIT 100 """ translate = utils.translator(lang) def each(res): lat = res["lat"] lon = res["lon"] error_id = res["id"] title = translate.select(res["title"]) subtitle = translate.select(res["subtitle"]) item = res["item"] or 0 text = title if subtitle: text += " - <br>" + subtitle return u"putAJAXMarker('%s', %f, %f, '%s', '%s');\n" % (error_id, lon, lat, text, item) response.content_type = 'text/plain; Charset=UTF-8' return query(db, sqlbase % (' AND '.join(clause), order), None, each)
def updates(db, lang): db.execute(""" SELECT analyser, COUNT(*), MIN(EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp)))/60/60/24 AS min_age, MAX(EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp)))/60/60/24 AS max_age, MIN(dynpoi_update_last.version) AS min_version, MAX(dynpoi_update_last.version) AS max_version FROM source JOIN dynpoi_update_last ON source.id = dynpoi_update_last.source WHERE dynpoi_update_last.version IS NOT NULL AND dynpoi_update_last.version NOT IN ('(None)', '(unknown)') GROUP BY analyser ORDER BY analyser """) summary = defaultdict(list) max_versions = None for res in db.fetchall(): (analyser, count, min_age, max_age, min_version, max_version) = res max_versions = max_version if max_version > max_versions else max_versions summary[analyser] = { 'count': count, 'min_age': min_age, 'max_age': max_age, 'max_version': '-'.join((max_version or '').split('-')[1:5]), 'min_version': '-'.join((min_version or '').split('-')[1:5]) } max_versions = '-'.join((max_versions or '').split('-')[1:5]) return template('control/updates_summary_by_analyser', translate=utils.translator(lang), summary=summary, max_versions=max_versions)
def updates(db, lang): db.execute(""" SELECT source.id, EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp)) AS age, source.country, source.analyser FROM source LEFT JOIN dynpoi_update_last ON source.id = dynpoi_update_last.source ORDER BY dynpoi_update_last.timestamp DESC """) liste = [] for res in db.fetchall(): (source, age, country, analyser) = (res[0], res[1], res[2], res[3]) if age: if age >= 0: # TRANSLATORS: days / hours / minutes since last source update, abbreviated to d / h / m txt = _("{day}d, {hour}h, {minute}m ago").format( day=int(age / 86400), hour=int(age / 3600) % 24, minute=int(age / 60) % 60) else: txt = _("in {day}d, {hour}h, {minute}m").format( day=int(-age / 86400), hour=int(-age / 3600) % 24, minute=int(-age / 60) % 60) liste.append((country, analyser, age, txt, source)) else: liste.append( (country, analyser, 1e10, _("never generated"), source)) liste.sort(lambda x, y: -cmp(x[2], y[2])) return template('control/updates', translate=utils.translator(lang), liste=liste)
def byuser_stats(db, lang): return template('byuser/byuser-stats', translate=utils.translator(lang), results=_users(db))
def index(db, lang, format=None): if request.path.endswith("false-positive"): title = _("False positives") gen = "false-positive" elif request.path.endswith("done"): title = _("Fixed issues") gen = "done" else: title = _("Informations") gen = "error" if not format in ('rss', 'gpx', 'josm'): format = None countries = query_meta._countries(db, lang) if format == None else None items = query_meta._items(db, lang) params = query._params() params.status = {"error":"open", "false-positive": "false", "done":"done"}[gen] params.limit = None if format == None and params.item: errors_groups = query._count(db, params, [ "dynpoi_class.item", "dynpoi_class.source", "dynpoi_class.class", "source.country", "source.analyser"], ["dynpoi_item"], [ "first(dynpoi_item.menu) AS menu", "first(dynpoi_class.title) AS title"], orderBy = True) total = 0 for res in errors_groups: if res["count"] != -1: total += res["count"] else: errors_groups = [] total = 0 params.limit = request.params.get('limit', type=int, default=100) if params.limit > 10000: params.limit = 10000 if (total > 0 and total < 1000) or params.limit: params.full = True errors = query._gets(db, params) if gen in ("false-positive", "done"): opt_date = "date" else: opt_date = "-1" else: opt_date = None errors = None if format == 'rss': response.content_type = 'application/rss+xml' tpl = 'errors/list.rss' elif format == 'gpx': response.content_type = 'application/gpx+xml' tpl = 'errors/list.gpx' elif format == 'josm': objects = [] for res in errors: if res["elems"]: elems = res["elems"].split("_") for e in elems: m = re.match(r"([a-z]+)([0-9]+)", e) if m: cur_type = m.group(1) objects.append(cur_type[0] + m.group(2)) response.status = 302 response.set_header('Location', 'http://localhost:8111/load_object?objects=%s' % ','.join(objects)) return else: tpl = 'errors/index' return template(tpl, countries=countries, items=items, errors_groups=errors_groups, total=total, errors=errors, query=request.query_string, country=params.country, item=params.item, translate=utils.translator(lang), gen=gen, opt_date=opt_date, title=title, website=utils.website)
def getBugsByUser(db, lang, user=None): user = user or request.params.get('user') clause, order = bboxRequest2Clause() sql_arg = {} limit = '' if user: clause.append("u.username = %(username)s") sql_arg['username'] = user else: num_points = request.params.get('points', type=int, default=100) if num_points != "all": limit = "LIMIT %d" % int(num_points) list2Clause(request.params.get('class'), clause, 'm.class') list2Clause(request.params.item, clause, 'm.item') list2Clause(request.params.not_item, clause, 'm.item NOT') sqlbase = """ SELECT marker.id, marker.item, marker.source, marker.class, marker.elems, marker.subclass, marker.lat, marker.lon, dynpoi_class.title as title, marker.subtitle as subtitle, dynpoi_update_last.timestamp, u.username FROM marker INNER JOIN dynpoi_class ON marker.source=dynpoi_class.source AND marker.class=dynpoi_class.class INNER JOIN dynpoi_update_last ON marker.source = dynpoi_update_last.source LEFT JOIN marker_elem u ON marker.id = u.marker_id WHERE %s ORDER BY dynpoi_update_last.timestamp DESC %s """ translate = utils.translator(lang) def each(res): lat = res["lat"] lon = res["lon"] error_id = res["id"] title = translate.select(res["title"]) subtitle = translate.select(res["subtitle"]) item = res["item"] or 0 return u'"%s", "%s", "%s", %f, %f, "%s", "%s", "%s"\n' % ( res["timestamp"], res["username"].decode('utf-8'), error_id, lon, lat, title, subtitle, item) response.content_type = 'text/plain; Charset=UTF-8' content = "# timestamp, username, error_id, lon, lat, title, subtitle, item\n" return content + query(db, sqlbase % (' AND '.join(clause), limit), sql_arg, each)
def translation(lang, name=None): translate = utils.translator(lang) return template('translation')
def contact(lang, name=None): translate = utils.translator(lang) return template('contact')
def copyright(lang, name=None): translate = utils.translator(lang) return template('copyright', translate=utils.translator(lang), main_project=utils.main_project, main_website=utils.main_website)
def contact(lang, name=None): translate = utils.translator(lang) return template('contact', translate=utils.translator(lang))
def copyright(lang, name=None): translate = utils.translator(lang) return template('copyright')
def index(db, user, lang): if request.query_string: redirect("./#" + request.query_string) tags = query_meta._tags(db) db.execute("SELECT item FROM dynpoi_item GROUP BY item;") all_items = map(lambda res: int(res[0]), db.fetchall()) categories = query_meta._items_3(db) item_tags = defaultdict(set) item_levels = {'1': set(), '2': set(), '3': set()} for categ in categories: for item in categ['items']: for level in item['levels']: item_levels[str(level['level'])].add(item['item']) if item['tags']: for tag in item['tags']: item_tags[tag].add(item['item']) item_levels['1,2'] = item_levels['1'] | item_levels['2'] item_levels['1,2,3'] = item_levels['1,2'] | item_levels['3'] urls = [] # TRANSLATORS: link to help in appropriate language if user: urls.append(("byuser", _("Issues by user"), "../byuser/")) urls.append(("relation_analyser", _("Relation analyser"), "http://analyser.openstreetmap.fr/")) # TRANSLATORS: link to source code urls.append(("statistics", _("Statistics"), "../control/update_matrix")) helps = [] helps.append((_("Contact"), "../contact")) helps.append( (_("Help on wiki"), _("http://wiki.openstreetmap.org/wiki/Osmose"))) helps.append((_("Copyright"), "../copyright")) helps.append((_("Sources"), "https://github.com/osm-fr?q=osmose")) helps.append((_("Translation"), "../translation")) sql = """ SELECT EXTRACT(EPOCH FROM ((now())-timestamp)) AS age FROM dynpoi_update_last ORDER BY timestamp LIMIT 1 OFFSET (SELECT COUNT(*)/2 FROM dynpoi_update_last) ; """ db.execute(sql) delay = db.fetchone() if delay and delay[0]: delay = delay[0] / 60 / 60 / 24 else: delay = 0 if user != None: if user: user_error_count = byuser._user_count(db, user.encode('utf-8')) else: # user == False user = '******' user_error_count = {1: 0, 2: 0, 3: 0} else: user_error_count = None return template('map/index', categories=categories, item_tags=item_tags, tags=tags, item_levels=item_levels, main_project=utils.main_project, urls=urls, helps=helps, delay=delay, languages_name=utils.languages_name, translate=utils.translator(lang), website=utils.website, remote_url_read=utils.remote_url_read, request=request, user=user, user_error_count=user_error_count)
def index(db, user, lang): if request.query_string: redirect("./#" + request.query_string) tags = query_meta._tags(db) db.execute("SELECT item FROM dynpoi_item GROUP BY item;") all_items = map(lambda res: int(res[0]), db.fetchall()) categories = query_meta._items_3(db) item_tags = defaultdict(set) item_levels = {'1': set(), '2': set(), '3': set()} for categ in categories: for item in categ['items']: for level in item['levels']: item_levels[str(level['level'])].add(item['item']) if item['tags']: for tag in item['tags']: item_tags[tag].add(item['item']) item_levels['1,2'] = item_levels['1'] | item_levels['2'] item_levels['1,2,3'] = item_levels['1,2'] | item_levels['3'] urls = [] # TRANSLATORS: link to help in appropriate language if user: urls.append(("byuser", _("Issues by user"), "../byuser/")) urls.append(("relation_analyser", _("Relation analyser"), "http://analyser.openstreetmap.fr/")) # TRANSLATORS: link to source code urls.append(("statistics", _("Statistics"), "../control/update_matrix")) helps = [] helps.append((_("Contact"), "../contact")) helps.append((_("Help on wiki"), _("http://wiki.openstreetmap.org/wiki/Osmose"))) helps.append((_("Copyright"), "../copyright")) helps.append((_("Sources"), "https://github.com/osm-fr?q=osmose")) helps.append((_("Translation"), "../translation")) sql = """ SELECT EXTRACT(EPOCH FROM ((now())-timestamp)) AS age FROM dynpoi_update_last ORDER BY timestamp LIMIT 1 OFFSET (SELECT COUNT(*)/2 FROM dynpoi_update_last) ; """ db.execute(sql) delay = db.fetchone() if delay and delay[0]: delay = delay[0]/60/60/24 else: delay = 0 if user != None: if user: user_error_count = byuser._user_count(db, user.encode('utf-8')) else: # user == False user = '******' user_error_count = {1: 0, 2: 0, 3: 0} else: user_error_count = None return template('map/index', categories=categories, item_tags=item_tags, tags=tags, item_levels=item_levels, main_project=utils.main_project, urls=urls, helps=helps, delay=delay, languages_name=utils.languages_name, translate=utils.translator(lang), website=utils.website, remote_url_read=utils.remote_url_read, request=request, user=user, user_error_count=user_error_count)
def index(lang): translate = utils.translator(lang) return template('index')
def translation(lang, name=None): translate = utils.translator(lang) return template('translation', translate=utils.translator(lang))
def copyright(lang, name=None): translate = utils.translator(lang) return template('copyright', main_project=utils.main_project, main_website=utils.main_website)
def index(db, lang, format=None): if request.path.endswith("false-positive"): title = _("False positives") gen = "false-positive" elif request.path.endswith("done"): title = _("Fixed issues") gen = "done" else: title = _("Informations") gen = "error" if not format in ('rss', 'gpx', 'josm'): format = None countries = query_meta._countries(db, lang) if format == None else None items = query_meta._items(db, lang) params = query._params() params.status = { "error": "open", "false-positive": "false", "done": "done" }[gen] params.limit = None if format == None and params.item: errors_groups = query._count( db, params, [ "dynpoi_class.item", "dynpoi_class.source", "dynpoi_class.class", "source.country", "source.analyser" ], ["dynpoi_item"], [ "min(dynpoi_item.menu::text)::hstore AS menu", "min(dynpoi_class.title::text)::hstore AS title" ], orderBy=True) total = 0 for res in errors_groups: if res["count"] != -1: total += res["count"] else: errors_groups = [] total = 0 params.limit = request.params.get('limit', type=int, default=100) if params.limit > 10000: params.limit = 10000 if (total > 0 and total < 1000) or params.limit: params.full = True errors = query._gets(db, params) if gen in ("false-positive", "done"): opt_date = "date" else: opt_date = "-1" else: opt_date = None errors = None if format == 'rss': response.content_type = 'application/rss+xml' tpl = 'errors/list.rss' elif format == 'gpx': response.content_type = 'application/gpx+xml' tpl = 'errors/list.gpx' elif format == 'josm': objects = [] for res in errors: if res["elems"]: elems = res["elems"].split("_") for e in elems: m = re.match(r"([a-z]+)([0-9]+)", e) if m: cur_type = m.group(1) objects.append(cur_type[0] + m.group(2)) response.status = 302 response.set_header( 'Location', 'http://localhost:8111/load_object?objects=%s' % ','.join(objects)) return else: tpl = 'errors/index' return template(tpl, countries=countries, items=items, errors_groups=errors_groups, total=total, errors=errors, query=request.query_string, country=params.country, item=params.item, translate=utils.translator(lang), gen=gen, opt_date=opt_date, title=title, website=utils.website, main_website=utils.main_website, remote_url_read=utils.remote_url_read)
def index(db, lang): # valeurs par défaut params = { "lat": 46.97, "lon": 2.75, "zoom": 6, "item": None, "level": 1, "source": '', "class": '', "username": '', "country": '', "tags": '', "fixable": None, } for p in ["lat", "lon", "zoom", "item", "level", "tags", "fixable"]: if request.cookies.get("last_" + p, default=None): params[p] = request.cookies.get("last_" + p) for p in ["lat", "lon", "zoom", "item", "useDevItem", "level", "source", "username", "class", "country", "tags", "fixable"]: if request.params.get(p, default=None): params[p] = request.params.get(p) for p in ["lat", "lon"]: params[p] = float(params[p]) for p in ["zoom"]: params[p] = int(params[p]) if not params.has_key("useDevItem"): params["useDevItem"] = "" tags = query_meta._tags(db, lang) tags_selected = {} tags_params = params["tags"].split(',') for t in tags: if t in tags_params: tags_selected[t] = " selected=\"selected\"" else: tags_selected[t] = "" fixable_selected = {} fixable_selected['online'] = " selected=\"selected\"" if params["fixable"] and params["fixable"] == "online" else "" fixable_selected['josm'] = " selected=\"selected\"" if params["fixable"] and params["fixable"] == "josm" else "" all_items = [] db.execute("SELECT item FROM dynpoi_item GROUP BY item;") for res in db.fetchall(): all_items.append(int(res[0])) active_items = check_items(params["item"], all_items) level_selected = {} for l in ("_all", "1", "2", "3", "1,2", "1,2,3"): level_selected[l] = "" if params["level"] == "": level_selected["1"] = " selected=\"selected\"" elif params["level"] in ("1", "2", "3", "1,2", "1,2,3"): level_selected[params["level"]] = " selected=\"selected\"" categories = query_meta._categories(db, lang) item_tags = {} item_levels = {"1": set(), "2": set(), "3": set()} for categ in categories: for err in categ["item"]: for l in err["levels"]: item_levels[str(l)].add(err["item"]) if err["tags"]: for t in err["tags"]: if not item_tags.has_key(t): item_tags[t] = set() item_tags[t].add(err["item"]) item_levels["1,2"] = item_levels["1"] | item_levels["2"] item_levels["1,2,3"] = item_levels["1,2"] | item_levels["3"] urls = [] # TRANSLATORS: link to help in appropriate language urls.append(("byuser", _("Issues by user"), "../byuser/")) urls.append(("relation_analyser", _("Relation analyser"), "http://analyser.openstreetmap.fr/")) # TRANSLATORS: link to source code urls.append(("statistics", _("Statistics"), "../control/update_matrix")) helps = [] helps.append((_("Contact"), "../contact")) helps.append((_("Help on wiki"), _("http://wiki.openstreetmap.org/wiki/Osmose"))) helps.append((_("Copyright"), "../copyright")) helps.append((_("Sources"), "https://github.com/osm-fr?query=osmose")) helps.append((_("Translation"), "../translation")) sql = """ SELECT EXTRACT(EPOCH FROM ((now())-timestamp)) AS age FROM dynpoi_update_last ORDER BY timestamp LIMIT 1 OFFSET (SELECT COUNT(*)/2 FROM dynpoi_update_last) ; """ db.execute(sql) delay = db.fetchone() if delay and delay[0]: delay = delay[0]/60/60/24 else: delay = 0 if request.session.has_key('user'): if request.session['user']: user = request.session['user']['osm']['user']['@display_name'] user_error_count = byuser._user_count(db, user.encode('utf-8')) else: user = '******' user_error_count = {1: 0, 2: 0, 3: 0} else: user = None user_error_count = None return template('map/index', categories=categories, lat=params["lat"], lon=params["lon"], zoom=params["zoom"], source=params["source"], username=params["username"], classs=params["class"], country=params["country"], item_tags=item_tags, tags_selected=tags_selected, tags=tags, fixable_selected=fixable_selected, item_levels=item_levels, level_selected=level_selected, active_items=active_items, useDevItem=params["useDevItem"], main_project=utils.main_project, urls=urls, helps=helps, delay=delay, languages_name=utils.languages_name, translate=utils.translator(lang), website=utils.website, request=request, assets=assets.environment, user=user, user_error_count=user_error_count)
def update(db, lang, source=None): sql = "SELECT source,timestamp,remote_url,remote_ip,version FROM dynpoi_update WHERE source=%d ORDER BY timestamp DESC;" % source db.execute(sql) return template('control/update', translate=utils.translator(lang), liste=db.fetchall())
def updates(db, lang): remote = request.params.get('remote') country = request.params.get('country') db.execute( """ SELECT DISTINCT ON (source.id) source.id, EXTRACT(EPOCH FROM ((now())-dynpoi_update_last.timestamp)) AS age, country, analyser FROM source JOIN dynpoi_update_last ON source.id = dynpoi_update_last.source WHERE """ + (""" RIGHT(MD5(remote_ip), 4) = %(remote)s AND """ if remote else "") + (""" source.country LIKE %(country)s AND """ if country else "") + """ true ORDER BY source.id ASC, dynpoi_update_last.timestamp DESC """, { "remote": remote, "country": country and country.replace("*", "%") }) keys = defaultdict(int) matrix = defaultdict(dict) stats_analyser = {} stats_country = {} for res in db.fetchall(): (source, age, country, analyser) = (res[0], res[1], res[2], res[3]) keys[country] += 1 matrix[analyser][country] = (age / 60 / 60 / 24, source) for analyser in matrix: min = max = None sum = 0 for country in matrix[analyser]: v = matrix[analyser][country][0] min = v if not min or v < min else min max = v if not max or v > max else max sum += v if not stats_country.has_key(country): min_c = v sum_c = v max_c = v n_c = 1 else: (min_c, sum_c, max_c, n_c) = stats_country[country] min_c = v if v < min_c else min_c max_c = v if v > max_c else max_c sum_c += v n_c += 1 stats_country[country] = [min_c, sum_c, max_c, n_c] stats_analyser[analyser] = [min, sum / len(matrix[analyser]), max] avg_country = {} for country in stats_country: stats_country[country][ 1] = stats_country[country][1] / stats_country[country][3] keys = sorted(keys.keys()) return template('control/updates_matrix', translate=utils.translator(lang), keys=keys, matrix=matrix, stats_analyser=stats_analyser, stats_country=stats_country)
def index(db, lang, format=None): if "false-positive" in request.path: title = _("False positives") gen = "false-positive" elif "done" in request.path: title = _("Fixed issues") gen = "done" else: title = _("Informations") gen = "error" if not format in ('rss', 'gpx', 'kml', 'josm', 'csv'): format = None countries = query_meta._countries(db, lang) if format == None else None items = query_meta._items(db, lang) params = query._params() params.status = {"error":"open", "false-positive": "false", "done":"done"}[gen] params.limit = None params.fixable = None if format == None and params.item: errors_groups = query._count(db, params, [ "dynpoi_class.item", "marker.source", "marker.class", "source.country", "source.analyser", "dynpoi_update_last.timestamp"], [ "dynpoi_item", "class"], [ "min(dynpoi_item.menu::text)::hstore AS menu", "min(class.title::text)::hstore AS title"], ) total = 0 for res in errors_groups: if res["count"] != -1: total += res["count"] else: errors_groups = [] total = 0 params.limit = request.params.get('limit', type=int, default=100) if params.limit > 10000: params.limit = 10000 if (total > 0 and total < 1000) or params.limit: params.full = True errors = query._gets(db, params) if gen in ("false-positive", "done"): opt_date = "date" else: opt_date = "-1" else: opt_date = None errors = None if format == 'rss': response.content_type = 'application/rss+xml' tpl = 'errors/list.rss' elif format == 'gpx': response.content_type = 'application/gpx+xml' tpl = 'errors/list.gpx' elif format == 'kml': response.content_type = 'application/vnd.google-earth.kml+xml' tpl = 'errors/list.kml' elif format == 'josm': objects = [] for res in errors: if res["elems"]: elems = res["elems"].split("_") for e in elems: m = re.match(r"([a-z]+)([0-9]+)", e) if m: cur_type = m.group(1) objects.append(cur_type[0] + m.group(2)) response.status = 302 response.set_header('Location', 'http://localhost:8111/load_object?objects=%s' % ','.join(objects)) return elif format == 'csv': output = StringIO.StringIO() writer = csv.writer(output) h = ['id', 'source', 'item', 'class', 'subclass', 'level', 'title', 'subtitle', 'country', 'analyser', 'timestamp', 'username', 'lat', 'lon', 'elems'] writer.writerow(h) for res in errors: writer.writerow(map(lambda a: res[a], h)) response.content_type = 'text/csv' return output.getvalue() else: tpl = 'errors/index' return template(tpl, countries=countries, items=items, errors_groups=errors_groups, total=total, errors=errors, query=request.query_string, country=params.country, item=params.item, level=params.level, translate=utils.translator(lang), gen=gen, opt_date=opt_date, title=title, website=utils.website, main_website=utils.main_website, remote_url_read=utils.remote_url_read)
def error(db, lang, err_id): data_type = { "N": "node", "W": "way", "R": "relation", "I": "infos"} # TRANSLATORS: link to tooltip help url_help = _("http://wiki.openstreetmap.org/wiki/Osmose/errors") translate = utils.translator(lang) (marker, columns_marker, elements, columns_elements, fixies, columns_fix) = _get(db, err_id) lat = str(marker["lat"]) lon = str(marker["lon"]) title = translate.select(marker["title"]) subtitle = translate.select(marker["subtitle"]) b_date = marker["timestamp"] or "" item = marker["item"] or 0 def expand_tags(tags, links, short = False): t = [] if short: for k in tags: t.append({"k": k}) else: for (k, v) in sorted(tags.items()): if links and links.has_key(k): t.append({"k": k, "v": v, "vlink": links[k]}) else: t.append({"k": k, "v": v}) return t elems = [] for elem in elements: if elem["data_type"]: tags = elem["tags"] try: links = t2l.checkTags(tags) except: links = {} tmp_elem = {data_type[elem["data_type"]]: True, "type": data_type[elem["data_type"]], "id": elem["id"], "tags": expand_tags(tags, links), "fixes": [], } for fix in fixies: if (fix["elem_data_type"] and fix["elem_data_type"] == elem["data_type"] and fix["elem_id"] == elem["id"]): tmp_elem["fixes"].append({"num": fix["diff_index"], "add": expand_tags(fix["tags_create"], {}), "mod": expand_tags(fix["tags_modify"], {}), "del": expand_tags(fix["tags_delete"], {}, True), }) elems.append(tmp_elem) new_elems = [] for fix in fixies: if fix["elem_data_type"]: found = False for e in elems: if (e["type"] == data_type[fix["elem_data_type"]] and e["id"] == fix[ "elem_id"]): found = True break if not found: new_elems.append({"num": fix["diff_index"], "add": expand_tags(fix["tags_create"], {}), "mod": expand_tags(fix["tags_modify"], {}), "del": expand_tags(fix["tags_delete"], {}, True), }) return { "lat":lat, "lon":lon, "minlat": float(lat) - 0.002, "maxlat": float(lat) + 0.002, "minlon": float(lon) - 0.002, "maxlon": float(lon) + 0.002, "error_id":err_id, "title":title, "subtitle":subtitle, "b_date":b_date.strftime("%Y-%m-%d"), "item":item, "elems":elems, "new_elems":new_elems, "elems_id":marker["elems"].replace("_",","), "url_help":url_help }
def index(lang): translate = utils.translator(lang) return template('index', translate=utils.translator(lang))
def index(db, lang, format=None): if "false-positive" in request.path: title = _("False positives") gen = "false-positive" elif "done" in request.path: title = _("Fixed issues") gen = "done" else: title = _("Informations") gen = "error" if not format in ('rss', 'gpx', 'kml', 'josm', 'csv'): format = None countries = query_meta._countries(db, lang) if format == None else None items = query_meta._items(db, lang) params = query._params() params.status = {"error":"open", "false-positive": "false", "done":"done"}[gen] params.limit = None params.fixable = None if format == None and params.item: errors_groups = query._count(db, params, [ "dynpoi_class.item", "marker.source", "marker.class", "source.country", "source.analyser", "dynpoi_update_last.timestamp"], [ "dynpoi_item", "class"], [ "min(dynpoi_item.menu::text)::hstore AS menu", "min(class.title::text)::hstore AS title"], orderBy = True) total = 0 for res in errors_groups: if res["count"] != -1: total += res["count"] else: errors_groups = [] total = 0 params.limit = request.params.get('limit', type=int, default=100) if params.limit > 10000: params.limit = 10000 if (total > 0 and total < 1000) or params.limit: params.full = True errors = query._gets(db, params) if gen in ("false-positive", "done"): opt_date = "date" else: opt_date = "-1" else: opt_date = None errors = None if format == 'rss': response.content_type = 'application/rss+xml' tpl = 'errors/list.rss' elif format == 'gpx': response.content_type = 'application/gpx+xml' tpl = 'errors/list.gpx' elif format == 'kml': response.content_type = 'application/vnd.google-earth.kml+xml' tpl = 'errors/list.kml' elif format == 'josm': objects = [] for res in errors: if res["elems"]: elems = res["elems"].split("_") for e in elems: m = re.match(r"([a-z]+)([0-9]+)", e) if m: cur_type = m.group(1) objects.append(cur_type[0] + m.group(2)) response.status = 302 response.set_header('Location', 'http://localhost:8111/load_object?objects=%s' % ','.join(objects)) return elif format == 'csv': output = StringIO.StringIO() writer = csv.writer(output) h = ['id', 'source', 'item', 'class', 'subclass', 'level', 'title', 'subtitle', 'country', 'analyser', 'timestamp', 'username', 'lat', 'lon', 'elems'] writer.writerow(h) for res in errors: writer.writerow(map(lambda a: res[a], h)) response.content_type = 'text/csv' return output.getvalue() else: tpl = 'errors/index' return template(tpl, countries=countries, items=items, errors_groups=errors_groups, total=total, errors=errors, query=request.query_string, country=params.country, item=params.item, level=params.level, translate=utils.translator(lang), gen=gen, opt_date=opt_date, title=title, website=utils.website, main_website=utils.main_website, remote_url_read=utils.remote_url_read)