def get_wikipedia_link(tags, locales):
""" Create a wikipedia link from a list of OSM tags. It scans for
wikipedia tags and reformats them to form a full URL.
Wikipedia tags with URLs already formed are not accepted.
"""
wikientries = TagStore(tags or {}).get_wikipedia_tags()
if not wikientries:
return None
for lang in locales:
if lang in wikientries:
title = quote(wikientries[lang].replace(' ', '_'))
return WIKIPEDIA_BASEURL.format(lang, title)
for k, v in wikientries.items():
url = WIKIPEDIA_TRANSLATEURL.format(k, quote(v.encode('utf8')),
lang)
try:
req = urllib.request.Request(
url, headers={'User-Agent': 'waymarkedtrails.org'})
data = urllib.request.urlopen(req).read().decode('utf-8')
data = json.loads(data)
except:
continue # oh well, we tried
(pgid, data) = data["query"]["pages"].popitem()
if 'langlinks' in data:
return data['langlinks'][0]['url']
else:
# given up to find a requested language
k, v = wikientries.popitem()
return WIKIPEDIA_BASEURL.format(k, quote(v.replace(' ', '_')))
def create_wikilink_response(self, res):
if res is None:
raise cherrpy.NotFound()
wikientries = TagStore(res['tags']).get_wikipedia_tags()
if not wikientries:
raise cherrypy.NotFound()
wikilink = 'https://%s.wikipedia.org/wiki/%s'
for lang in cherrypy.request.locales:
if lang in wikientries:
title = urllib.parse.quote(wikientries[lang].replace(' ', '_'))
raise cherrypy.HTTPRedirect(wikilink % (lang, title))
for k, v in wikientries.items():
url = "https://%s.wikipedia.org/w/api.php?action=query&prop=langlinks&titles=%s&llprop=url&&lllang=%s&format=json" % (
k, urllib.parse.quote(v.encode('utf8')), lang)
try:
req = urllib.request.Request(
url,
headers={'User-Agent': 'Python-urllib/2.7 Routemaps'})
data = urllib.request.urlopen(req).read().decode('utf-8')
data = jsonlib.loads(data)
except:
continue # oh well, we tried
(pgid, data) = data["query"]["pages"].popitem()
if 'langlinks' in data:
raise cherrypy.HTTPRedirect(data['langlinks'][0]['url'])
else:
# given up to find a requested language
k, v = wikientries.popitem()
raise cherrypy.HTTPRedirect(
wikilink % (k, urllib.parse.quote(v.replace(' ', '_'))))
def transform(self, obj):
tags = TagStore(obj['tags'])
# filter by subtype
if self.config.subtype is not None:
booltags = tags.get_booleans()
if len(booltags) > 0:
if not booltags.get(self.config.subtype, False):
return None
else:
if self.config.require_subtype:
return None
outtags = {'name': tags.get('name'), 'ele': None}
if 'ele' in tags:
m = self.elepattern.search(tags['ele'])
if m:
outtags['ele'] = m.group(0)
# XXX check for ft
if self.srid == self.src.c.geom.type.srid:
outtags['geom'] = obj['geom']
else:
outtags['geom'] = obj['geom'].ST_Transform(self.srid)
return outtags
def basic_tag_transform(tags: TagStore, config):
difficulty = tags.get('piste:difficulty')
difficulty = config.difficulty_map.get(difficulty, 0)
return dict(intnames=tags.get_prefixed('name:'),
name=tags.firstof('piste:name', 'name'),
ref=tags.firstof('piste:ref', 'ref'),
difficulty=difficulty,
piste=config.piste_type.get(tags.get('piste:type'), 0))
def _construct_row(self, obj, conn):
tags = TagStore(obj['tags'])
outtags, difficulty = basic_tag_transform(obj['id'], tags, self.config)
# we don't support hierarchy at the moment
outtags['top'] = True
# geometry
geom = build_route_geometry(conn, obj['members'], self.ways, self.data)
if geom is None:
return None
if geom.geom_type not in ('MultiLineString', 'LineString'):
raise RuntimeError("Bad geometry %s for %d" %
(geom.geom_type, obj['id']))
# if the route is unsorted but linear, sort it
if geom.geom_type == 'MultiLineString':
fixed_geom = linemerge(geom)
if fixed_geom.geom_type == 'LineString':
geom = fixed_geom
outtags['geom'] = from_shape(geom, srid=self.c.geom.type.srid)
outtags['symbol'] = write_symbol(self.shield_fab, tags, difficulty,
self.config.symbol_datadir)
outtags['id'] = obj['id']
return outtags
def symbols(self, **params):
from db.common.symbols import ShieldFactory
factory = ShieldFactory(
'SwissMobile',
'JelRef',
'KCTRef',
'ItalianHikingRefs',
'OSMCSymbol',
'Nordic',
'Slopes',
'ShieldImage',
'TextColorBelow',
'ColorBox',
'TextSymbol',
)
sym = factory.create(TagStore(params), params.get('_network', ''), 10)
if sym is None:
raise cherrypy.NotFound()
with NamedTemporaryFile() as f:
sym.write_image(f.name)
factory._mangle_svg(f.name)
with open(f.name, 'rb') as f:
res = f.read()
cherrypy.response.headers['Content-Type'] = 'image/svg+xml'
return res
def transform_tags(self, obj):
tags = TagStore(obj['tags'])
outtags, difficulty = _basic_tag_transform(obj['id'], tags)
outtags['symbol'] = shield_fab.create_write(tags, '', difficulty)
return outtags
def _process_next(self, obj):
tags = self.transform_tags(obj['id'], TagStore(obj['tags']))
if tags is not None:
tags[self.id_column.name] = obj['id']
tags['geom'] = str(obj['geom'])
self.thread.compiled_insert.execute(tags)
def create_details_response(self, res):
if res is None:
raise cherrypy.NotFound()
loctags = TagStore.make_localized(res['tags'],
cherrypy.request.locales)
cfg = cherrypy.request.app.config
ret = api.common.RouteDict(res)
ret['type'] = res['type'] if res.has_key('type') else 'relation'
ret['symbol_url'] = '%s/symbols/%s/%s.svg' % (
cfg['Global']['MEDIA_URL'], cfg['Global']['BASENAME'],
str(res['symbol']))
ret['mapped_length'] = int(res['length'])
ret.add_if('official_length',
loctags.get_length('distance', 'length', unit='m'))
for tag in ('operator', 'note', 'description'):
ret.add_if(tag, loctags.get(tag))
ret.add_if('url', loctags.get_url())
ret.add_if('wikipedia', loctags.get_wikipedia_tags())
ret['bbox'] = to_shape(res['bbox']).bounds
if hasattr(self, '_hierarchy_list'):
for name, val in (('subroutes', True), ('superroutes', False)):
ret.add_if(name, self._hierarchy_list(ret['id'], val))
ret['tags'] = res['tags']
return ret
def mkshield(self):
route = self.tables.routes
sway = self.tables.ways
rel = self.osmdata.relation.data
way = self.osmdata.way.data
todo = ((route, select([rel.c.tags
]).where(rel.c.id == route.data.c.id)),
(sway, select([way.c.tags]).where(way.c.id == sway.data.c.id)))
donesyms = set()
with self.engine.begin() as conn:
for src, sel in todo:
for r in conn.execution_options(
stream_results=True).execute(sel):
tags = TagStore(r["tags"])
t, difficulty = piste_tag_transform(0, tags)
sym = src.symbols.create(tags, '', difficulty)
if sym is not None:
symid = sym.get_id()
if symid not in donesyms:
donesyms.add(symid)
src.symbols.write(sym, True)
def transform_tags(self, obj):
tags = TagStore(obj['tags'])
outtags, difficulty = basic_tag_transform(obj['id'], tags, self.config)
outtags['symbol'] = write_symbol(self.shield_fab, tags, difficulty,
self.config.symbol_datadir)
return outtags
def _construct_row(self, obj, conn):
tags = TagStore(obj['tags'])
outtags = basic_tag_transform(TagStore(obj['tags']), self.config)
# we don't support hierarchy at the moment
outtags['top'] = True
# geometry
geom = make_geometry(conn, obj['members'], self.ways, self.data)
if geom is None:
return None
outtags['geom'] = geom
outtags['symbol'] = write_symbol(self.shield_fab, tags,
outtags['difficulty'],
self.config.symbol_datadir)
outtags['id'] = obj['id']
return outtags
def _construct_row(self, obj, conn):
tags = TagStore(obj['tags'])
is_node_network = tags.get('network:type') == 'node_network'
outtags = RouteRow(id=obj['id'],
name=tags.get('name'),
ref=tags.get('ref'),
intnames=tags.get_prefixed('name:'),
itinerary=make_itinerary(tags))
if 'symbol' in tags:
outtags.intnames['symbol'] = tags['symbol']
if is_node_network:
outtags.level = Network.LOC.min()
elif 'network' in tags:
outtags.level = self._compute_route_level(tags['network'])
# child relations
members, relids = self._filter_members(obj['id'], obj['members'])
# geometry
geom = make_geometry(conn, members, self.ways, self.data)
if geom is None:
return None
# find the country
outtags.country = self._find_country(relids, geom)
# create the symbol
outtags.symbol = self._write_symbol(
tags, outtags.country,
Network.from_int(outtags.level).name)
# custom filter callback
if self.config.tag_filter is not None:
self.config.tag_filter(outtags, tags)
if outtags.network is None and is_node_network:
outtags.network = 'NDS'
if 'network' in tags and not is_node_network:
h = self.rtree.data
r = self.rels.data
sel = sa.select([sa.text("'a'")])\
.where(h.c.child == obj['id'])\
.where(r.c.id == h.c.parent)\
.where(h.c.depth == 2)\
.where(r.c.tags['network'].astext == tags['network'])\
.limit(1)
top = self.thread.conn.scalar(sel)
outtags.top = (top is None)
outtags = dataclasses.asdict(outtags)
outtags['geom'] = geom
return outtags
def _write_shields(self, source, subset, donesyms):
with self.engine.begin() as conn:
for r in conn.execution_options(stream_results=True).execute(subset):
tags = TagStore(r["tags"])
_, difficulty = self.tables.routes.basic_tag_transform(0, tags)
sym = source.symbols.create(tags, '', difficulty)
if sym is not None:
symid = sym.get_id()
if symid not in donesyms:
donesyms.add(symid)
source.symbols.write(sym, True)
def transform(self, obj):
tags = TagStore(obj['tags'])
if self.config.node_tag not in tags:
return None
outtags = {'name': tags[self.config.node_tag]}
if self.srid == self.src.c.geom.type.srid:
outtags['geom'] = obj['geom']
else:
outtags['geom'] = obj['geom'].ST_Transform(self.srid)
return outtags
def _add_details(self, row, locales):
loctags = TagStore.make_localized(row['tags'], locales)
self.content['mapped_length'] = int(row['length'])
self._add_optional('official_length', row, None,
loctags.get_length('distance', 'length', unit='m'))
for tag in ('operator', 'note', 'description'):
self._add_optional(tag, row, None, loctags.get(tag))
self._add_optional('url', row, None, loctags.get_url())
self._add_optional('wikipedia', row, None,
loctags.get_wikipedia_tags() or None)
self.content['bbox'] = to_shape(row['bbox']).bounds
self.content['tags'] = row['tags']
def index(self, oid, **params):
cfg = cherrypy.request.app.config
mapdb = cfg['DB']['map']
r = mapdb.tables.guideposts.data
o = mapdb.osmdata.node.data
sel = sa.select([r.c.name, r.c.ele,
r.c.geom.ST_X().label('lon'),
r.c.geom.ST_Y().label('lat'),
o.c.tags])\
.where(r.c.id == oid)\
.where(o.c.id == oid)
res = cherrypy.request.db.execute(sel).first()
if res is None:
raise cherrypy.NotFound()
ret = OrderedDict()
loctags = TagStore.make_localized(res['tags'],
cherrypy.request.locales)
ret['type'] = 'guidepost'
ret['id'] = oid
if 'name' in loctags:
ret['name'] = loctags['name']
if res['name'] and res['name'] != ret['name']:
ret['local_name'] = res['name']
if res['ele'] is not None:
ret['ele'] = loctags.get_length('ele', unit='m', default='m')
for tag in ('ref', 'operator', 'description', 'note'):
if tag in loctags:
ret[tag] = loctags[tag]
if 'image' in loctags:
imgurl = loctags['image']
if imgurl.startswith('http://') or imgurl.startswith('https://'):
# HTML injection paranoia
imgurl.replace('"', '%22')
imgurl.replace("'", '%27')
ret['image'] = imgurl
ret['tags'] = res['tags']
ret['y'] = res['lat']
ret['x'] = res['lon']
return ret
def mkshield(self):
route = self.tables.routes
rel = self.osmdata.relation.data
sel = select([rel.c.tags, route.data.c.country, route.data.c.level])\
.where(rel.c.id == route.data.c.id)
donesyms = set()
with self.engine.begin() as conn:
for r in conn.execution_options(stream_results=True).execute(sel):
sym = route.symbols.create(TagStore(r["tags"]), r["country"],
r["level"])
if sym is not None:
symid = sym.get_id()
if symid not in donesyms:
donesyms.add(symid)
route.symbols.write(sym, True)
def add_row_data(self, row, locales):
loctags = TagStore.make_localized(row['tags'], locales)
if 'name' in loctags:
self.content['name'] = loctags['name']
if row['name'] and row['name'] != self.content['name']:
self.content['local_name'] = row['name']
self.add_if('ele', row['ele'])
for tag in ('ref', 'operator', 'description', 'note'):
self.add_if(tag, loctags.get(tag))
self.add_if('image', loctags.get_url(keys=['image']))
for key in ('tags', 'x', 'y'):
self.content[key] = row[key]
return self
def _construct_row(self, obj, conn):
if hasattr(self, 'transform_tags'):
cols = self.transform_tags(obj['way_id'], TagStore(obj['tags']))
if cols is None:
return None
else:
cols = {}
if self.osmdata is not None:
points = self.osmdata.get_points(obj['nodes'], conn)
if len(points) <= 1:
return
if self.srid == 3857:
points = [p.to_mercator() for p in points]
cols['geom'] = from_shape(sgeom.LineString(points), srid=self.srid)
cols['id'] = obj['way_id']
cols['rels'] = sorted(obj['rels'])
cols['nodes'] = obj['nodes']
return cols
def _update_handle_changed_ways(self, engine):
""" Handle changes to way tags, added and removed nodes and moved nodes.
"""
with_tags = hasattr(self, 'transform_tags')
with_geom = self.osmdata is not None
# Get old rows where nodes and tags have changed and new node set.
d = self.data
w = self.way_src.data
wc = self.way_src.change
# ids of ways that have changed directly
sql_idchg = sa.select([d.c.id])\
.where(d.c.id == wc.c.id).where(wc.c.action != 'D')
# ids of ways where nodes have changed (only geometry mode)
if with_geom:
nc = self.osmdata.node.cc.id
sql_ndchg = sa.select([d.c.id])\
.where(d.c.nodes.overlap(array([nc])))
sql_idchg = sa.union(sql_idchg, sql_ndchg)
sql_idchg = sql_idchg.alias('ids')
waynode_sql = sa.select([w.c.nodes]).where(w.c.id == d.c.id)
cols = [d, waynode_sql.as_scalar().label('new_nodes')]
if with_tags:
waytag_sql = sa.select([w.c.tags]).where(w.c.id == d.c.id)
cols.append(waytag_sql.as_scalar().label('new_tags'))
sql = sa.select(cols).where(sql_idchg.c.id == d.c.id)
inserts = []
deletes = []
changeset = {}
for obj in engine.execute(sql):
oid = obj['id']
if obj['new_nodes'] is None:
deletes.append({'oid': oid})
changeset[oid] = 'D'
continue
changed = False
if with_tags:
cols = self.transform_tags(oid, TagStore(obj['new_tags']))
if cols is None:
deletes.append({'oid': oid})
changeset[oid] = 'D'
continue
# check if there are actual tag changes
for k, v in cols.items():
if str(obj[k]) != str(v):
changed = True
break
else:
cols = {}
# Always rebuild the geometry when with_geom as nodes might have
# moved.
if with_geom:
# TODO only look up new/changed nodes
points = self.osmdata.get_points(obj['new_nodes'], engine)
if len(points) <= 1:
deletes.append({'oid': oid})
changeset[oid] = 'D'
continue
if self.srid == 3857:
points = [p.to_mercator() for p in points]
new_geom = sgeom.LineString(points)
cols['geom'] = from_shape(new_geom, srid=self.srid)
changed = changed or (new_geom != to_shape(obj['geom']))
elif obj['nodes'] != obj['new_nodes']:
changed = True
if changed:
cols['nodes'] = obj['new_nodes']
cols['id'] = oid
cols['rels'] = obj['rels']
inserts.append(cols)
changeset[oid] = 'M'
if len(inserts):
engine.execute(self.upsert_data().values(inserts))
if len(deletes):
engine.execute(
self.data.delete().where(self.c.id == sa.bindparam('oid')),
deletes)
return changeset
def _construct_row(self, obj, conn):
tags = TagStore(obj['tags'])
outtags = RouteRow(obj['id'])
# determine name and level
for k, v in tags.items():
if k in ('name', 'ref'):
outtags[k] = v
elif k.startswith('name:'):
outtags.intnames[k[5:]] = v
elif k == 'network':
outtags.level = self._compute_route_level(v)
if tags.get('network:type') == 'node_network':
outtags.level = Network.LOC.min()
# child relations
relids = [ r['id'] for r in obj['members'] if r['type'] == 'R']
members = obj['members']
if len(relids) > 0:
# Is this relation part of a cycle? Then drop the relation members
# to not get us in trouble with geometry building.
h1 = self.rtree.data.alias()
h2 = self.rtree.data.alias()
sql = sa.select([h1.c.parent])\
.where(h1.c.parent == obj['id'])\
.where(h1.c.child == h2.c.parent)\
.where(h2.c.child == obj['id'])
if (self.thread.conn.execute(sql).rowcount > 0):
members = [ m for m in obj['members'] if m['type'] == 'W' ]
relids = []
# geometry
geom = build_route_geometry(conn, members, self.ways, self.data)
if geom is None:
return None
if geom.geom_type not in ('MultiLineString', 'LineString'):
raise RuntimeError("Bad geometry %s for %d" % (geom.geom_type, obj['id']))
# if the route is unsorted but linear, sort it
if geom.geom_type == 'MultiLineString':
fixed_geom = linemerge(geom)
if fixed_geom.geom_type == 'LineString':
geom = fixed_geom
outtags.geom = from_shape(geom, srid=self.data.c.geom.type.srid)
# find the country
if len(relids) > 0:
sel = sa.select([self.c.country], distinct=True)\
.where(self.c.id.in_(relids))
else:
c = self.countries
sel = sa.select([c.column_cc()], distinct=True)\
.where(c.column_geom().ST_Intersects(outtags.geom))
cur = self.thread.conn.execute(sel)
# should be counting when rowcount > 1
if cur.rowcount >= 1:
cntry = cur.scalar()
else:
cntry = None
outtags.country = cntry
sym = self.symbols.create(tags, cntry,
style=Network.from_int(outtags.level).name)
if sym is None:
outtags.symbol = 'None'
else:
outtags.symbol = sym.uuid()
sym.to_file(os.path.join(self.config.symbol_datadir,
outtags.symbol + '.svg'), format='svg')
# custom filter callback
if self.config.tag_filter is not None:
self.config.tag_filter(outtags, tags)
if outtags.network is None:
if tags.get('network:type') == 'node_network':
outtags.network = 'NDS'
if outtags.top is None:
if 'network' in tags and tags.get('network:type') != 'node_network':
h = self.rtree.data
r = self.rels.data
sel = sa.select([sa.text("'a'")]).where(h.c.child == obj['id'])\
.where(r.c.id == h.c.parent)\
.where(h.c.depth == 2)\
.where(r.c.tags['network'].astext == tags['network'])\
.limit(1)
top = self.thread.conn.scalar(sel)
outtags.top = (top is None)
else:
outtags.top = True
return outtags
