def _get_err(self, tag, name):
initialName = name
for x in [u"&", u"'", u""", u"/", u")", u"-", u"\"", u";", u".", u":", u"+", u"?", u"!", u",", u"|", u"*", u"°", u"_", u"="]:
name = name.replace(x, " ")
if self.apostrophe:
name = self.apostrophe.sub(' ', name)
for WordComplet in name.split(" "):
if WordComplet in self.DictCommonWords: continue
elif WordComplet in self.DictKnownWords: continue
elif WordComplet in self.DictCorrections:
if self.DictCorrections[WordComplet]:
return {"class": 703, "subclass": stablehash(tag), "fix": {"name": initialName.replace(WordComplet, self.DictCorrections[WordComplet])}}
else:
raise Exception("Could not find correction for %s" % WordComplet)
else:
for x in self.DictEncoding:
if x in WordComplet:
return {"class": 704, "subclass": stablehash(tag), "fix": {"name": initialName.replace(x, self.DictEncoding[x])}}
#if WordComplet in self.DictUnknownWords: continue
if "0" in WordComplet: continue
if "1" in WordComplet: continue
if "2" in WordComplet: continue
if "3" in WordComplet: continue
if "4" in WordComplet: continue
if "5" in WordComplet: continue
if "6" in WordComplet: continue
if "7" in WordComplet: continue
if "8" in WordComplet: continue
if "9" in WordComplet: continue
self.DictUnknownWords.add(WordComplet)
def check(self, tags):
err = []
for tag in self.URL_TAGS:
if tag not in tags:
continue
url = tags[tag]
stripped = False
if ' ' in url:
url = url.strip()
if ' ' in url:
# We don't know how to fix such a URL: Remove everything
# after the space? Encode the space?
err.append({"class": 30931, "subclass": stablehash(tag), "text": self._bad_url(tag, tags)})
continue
stripped = True
if self.HasScheme.match(url):
if stripped:
err.append({"class": 30931, "fix": {tag: url}})
continue
elif url.startswith('://'):
url = url[3:]
elif ':' in url or '//' in url:
# The URL already contains some sort of broken scheme
# so it's too complex for us to fix
err.append({"class": 30932, "subclass": stablehash(tag), "text": self._bad_url(tag, tags)})
continue
err.append({"class": 30932, "subclass": stablehash(tag), "fix": [
{tag: "https://" + url},
{tag: "http://" + url}
]})
return err
def node(self, data, tags):
err = []
for k in tags:
if k in self._update_ks:
err.append({"class": self._update_ks[k][1], "subclass": stablehash(k), "text": T_f(u"tag key: {0} => {1} (rule ks)", k, self._update_ks[k][0])})
if k in self._update_ks_vs and tags[k] in self._update_ks_vs[k]:
err.append({"class": self._update_ks_vs[k][tags[k]][1], "subclass": stablehash(u"%s=%s" % (k,tags[k])), "text": T_f(u"tag value: {0}={1} => {2} (rule ks_vs)", k, tags[k],self._update_ks_vs[k][tags[k]][0])})
if k in self._update_ks_vr:
for v in self._update_ks_vr[k]:
if v.match(tags[k]):
err.append({"class": self._update_ks_vr[k][v][1], "subclass": stablehash(u"%s=%s" % (k,tags[k])), "text": T_f(u"tag value: {0}={1} => {2} (rule ks_vr)", k, tags[k],self._update_ks_vr[k][v][0])})
for kk in tags:
for k in self._update_kr:
if k.match(kk):
err.append({"class": self._update_kr[k][1], "subclass": stablehash(kk), "text": T_f(u"tag key: {0} => {1} (rule kr)", kk, self._update_kr[k][0])})
for k in self._update_kr_vs:
if k.match(kk):
if tags[kk] in self._update_kr_vs[k]:
err.append({"class": self._update_kr_vs[k][tags[kk]][1], "subclass": stablehash(u"%s=%s" % (kk,tags[kk])), "text": T_f(u"tag value: {0}={1} => {2} (rule kr_vs)", kk, tags[kk], self._update_kr_vs[k][tags[kk]][0])})
for k in self._update_kr_vr:
if k.match(kk):
for v in self._update_kr_vr[k]:
if v.match(tags[kk]):
err.append({"class": self._update_kr_vr[k][v][1], "zsubclass": stablehash(u"%s=%s" % (kk,tags[kk])), "text": T_f(u"tag value: {0}={1} => {2} (rule ks_vr)", kk, tags[kk], self._update_kr_vr[k][v][0])})
return err
def node(self, data, tags):
err = []
for k in set(tags).intersection(self.DeprecatedSet):
if None in self.Deprecated[k]:
err.append({"class": 4010, "subclass": stablehash(k), "text": T_("Tag %(tag)s is deprecated: %(depr)s", {"tag": k, "depr": self.Deprecated[k][None]})})
elif tags[k] in self.Deprecated[k]:
err.append({"class": 40102, "subclass": stablehash(k), "text": T_("Tag %(tag)s=%(value)s is deprecated: %(depr)s", {"tag": k, "value": tags[k], "depr": self.Deprecated[k][tags[k]]})})
return err
def __init__(self, config, logger=None):
Analyser_Osmosis.__init__(self, config, logger)
highway_values = ("motorway", "trunk", "primary", "secondary")
if self.config.options and "osmosis_way_approximate" in self.config.options and self.config.options[
"osmosis_way_approximate"].get("highway"):
highway_values = self.config.options[
"osmosis_way_approximate"].get("highway")
self.tags = (
(10, "railway", ("rail", )),
(20, "waterway", ("river", )),
(30, "highway", highway_values),
)
for t in self.tags:
self.classs_change[t[0]] = {
"item": "1190",
"level": 3,
"tag": ["geom", "highway", "railway", "fix:imagery"],
"desc": T_f(u"Approximate {0}", t[1])
}
self.callback10 = lambda res: {
"class": res[4],
"subclass": stablehash(res[3]),
"data": [self.way_full, self.positionAsText],
"text": T_f(u"{0} deviation of {1}m", res[3], res[2])
}
def __init__(self, config, logger = None):
Analyser_Osmosis.__init__(self, config, logger)
self.classs_change[1] = {"item":"1170", "level": 3, "tag": ["relation", "geom", "fix:chair"], "desc": T_(u"Double inner polygon") }
self.classs_change[2] = {"item":"1170", "level": 2, "tag": ["relation", "multipolygon", "fix:chair"], "desc": T_(u"Inconsistant multipolygon nature with members nature") }
self.classs_change[3] = {"item":"1170", "level": 2, "tag": ["relation", "multipolygon", "fix:chair"], "desc": T_(u"Inconsistant multipolygon member nature") }
self.classs_change[4] = {"item":"1170", "level": 1, "tag": ["relation", "geom", "fix:chair"], "desc": T_(u"Should be polygon, part of multipolygon or not having area tag") }
self.callback10 = lambda res: {"class":1, "data":[self.way_full, self.way_full, self.positionAsText]}
self.callback20 = lambda res: {"class":2, "subclass":stablehash(res[11]), "data":[self.relation_full, self.way_full, self.positionAsText],
"text": {"en": u", ".join(map(lambda k: "%s=(%s,%s)"%k, filter(lambda k: k[1], (("landuse",res[3],res[4]), ("natural",res[5],res[6]), ("waterway",res[7],res[8]), ("building",res[9],res[10])))))}
}
self.callback30 = lambda res: {"class":3, "subclass":1, "data":[self.relation_full, self.positionAsText],
"text": {"en": u", ".join(map(lambda k: "%s=(%s)"%k, filter(lambda k: k[1], (("landuse",res[2]), ("natural",res[3]), ("waterway",res[4]), ("building",res[5])))))}
}
self.callback40 = lambda res: {"class":4, "subclass":stablehash(res[9]), "data":[self.way_full, self.positionAsText],
"text": {"en": u", ".join(map(lambda k: "%s=%s"%k, filter(lambda k: k[1], (("area",res[2]), ("landuse",res[3]), ("natural",res[4]), ("waterway",res[5]), ("leisure",res[6]), ("amenity",res[7]), ("building",res[8])))))}
}
def node(self, data, tags):
err = []
for k in set(tags).intersection(self.DeprecatedSet):
if None in self.Deprecated[k]:
err.append({
"class": 4010,
"subclass": stablehash(k),
"text": T_('The tag `{0}` is deprecated in favour of {1}', k, self.Deprecated[k][None])
})
elif tags[k] in self.Deprecated[k]:
err.append({
"class": 40102,
"subclass": stablehash(k),
"text": T_('The tag `{0}` is deprecated in favour of {1}', "=".join([k, tags[k]]), self.Deprecated[k][tags[k]])
})
return err
def node(self, data, tags):
if u"name" not in tags:
return
if (u"highway" not in tags) and (u"waterway" not in tags) and (u"place" not in tags):
return
words = []
name = tags[u"name"]
name_subst = self.apply_special_subst(name)
split = self._split(name_subst)
for i in range(0, len(split), 2):
split[i] = self.remove_special_subst(split[i])
splitfix = list(split)
if split and split[0] and split[0][0] in self.minus:
words.append(split[0])
splitfix[0] = split[0].capitalize()
for i in range(0, len(split), 2):
word = split[i]
if word in self.special:
continue
if word[0] in self.minus:
words.append(word)
splitfix[i] = split[i].capitalize()
if words:
return {"class": 906, "subclass": stablehash(','.join(words)), "text": T_(u"Missing capital letter for: %s", u", ".join(sorted(set(words)))),
"fix": {"name": "".join(splitfix)} }
return
def way(self, data, tags, nds):
if not "highway" in tags:
return
err = []
sides = list(map(lambda tag: tag[len(self.parking_lane):].split(":")[0], filter(lambda tag: tag.startswith(self.parking_lane), tags)))
n_sides = len(sides)
sides = [i for i in sides if i not in ("left", "right", "both")]
conditions = map(lambda tag: ":".join(tag.split(":")[0:3]).replace(":condition:", ":lane:"), filter(lambda tag: tag.startswith(self.parking_condition), tags))
for c in conditions:
if c not in tags:
err.append({"class": 31616})
break
if n_sides == 0:
return err
if len(sides) > 0:
err.append({"class": 31611})
if ("parking:lane:right" in tags or "parking:lane:left" in tags) and "parking:lane:both" in tags:
err.append({"class": 31614})
for side in ("parking:lane:right", "parking:lane:left", "parking:lane:both"):
if side in tags and tags[side] not in ("parallel", "diagonal", "perpendicular", "marked", "no_parking", "no_stopping", "fire_lane"):
err.append({"class": 31615, "subclass": stablehash(side)})
return err
def analyser_osmosis_common(self):
self.run(
sql10, lambda res: {
"class": 1,
"subclass": stablehash(res[0]),
"data": [self.positionAsText]
})
def node(self, data, tags):
err = []
keys = tags.keys()
keys = set(keys) & self.SimpleValuedTag
for k in keys:
if ';' in tags[k]:
err.append({"class": 3070, "subclass": stablehash(k), "text": T_("Multiple values on tag: %(key)s=%(val)s", {"key": k, "val": tags[k]})})
return err
def init(self, logger):
Plugin.init(self, logger)
country = self.father.config.options.get("country") if self.father else None
language = self.father.config.options.get("language") if self.father else None
if isinstance(language, list):
language = None
elif language:
language = language.split('_')[0]
self._update_ks = {}
self._update_kr = {}
self._update_ks_vs = defaultdict(dict)
self._update_kr_vs = defaultdict(dict)
self._update_ks_vr = defaultdict(dict)
self._update_kr_vr = defaultdict(dict)
reline = re.compile("^\|([^|]*)\|\|([^|]*)\|\|([^|]*)\|\|([^|]*).*")
# récupération des infos depuis https://wiki.openstreetmap.org/index.php?title=User:FrViPofm/TagwatchCleaner
data = urlread(u"https://wiki.openstreetmap.org/index.php?title=User:FrViPofm/TagwatchCleaner&action=raw", 1)
data = data.split("\n")
for line in data:
for res in reline.findall(line):
only_for = res[3].strip()
if only_for in (None, '', country, language) or (country and country.startswith(only_for)):
r = res[1].strip()
c0 = res[2].strip()
tags = ["fix:chair"] if c0 == "" else [c0, "fix:chair"]
c = stablehash(c0)
self.errors[c] = self.def_class(item = 3030, level = 2, tags = tags,
title = {'en': c0},
detail = T_(
'''Simple and frequent errors, the list is available
[here](https://wiki.openstreetmap.org/wiki/User:FrViPofm/TagwatchCleaner).'''))
if u"=" in res[0]:
k = res[0].split(u"=")[0].strip()
v = res[0].split(u"=")[1].strip()
if self.quoted(k):
k = self.quoted2re(k)
if self.quoted(v):
self._update_kr_vr[k][self.quoted2re(v)] = [r, c]
else:
self._update_kr_vs[k][v] = [r, c]
else:
if self.quoted(v):
self._update_ks_vr[k][self.quoted2re(v)] = [r, c]
else:
self._update_ks_vs[k][v] = [r, c]
else:
if self.quoted(res[0]):
self._update_kr[self.quoted2re(res[0])] = [r, c]
else:
self._update_ks[res[0]] = [r, c]
def __init__(self, config, logger = None):
Analyser_Osmosis.__init__(self, config, logger)
highway_values = ("motorway", "trunk", "primary", "secondary")
if self.config.options and "osmosis_way_approximate" in self.config.options and self.config.options["osmosis_way_approximate"].get("highway"):
highway_values = self.config.options["osmosis_way_approximate"].get("highway")
self.tags = ( (10, "railway", ("rail",)),
(20, "waterway", ("river",)),
(30, "highway", highway_values),
)
for t in self.tags:
self.classs_change[t[0]] = {"item":"1190", "level": 3, "tag": ["geom", "highway", "railway", "fix:imagery"], "desc": T_f(u"Approximate {0}", t[1]) }
self.callback10 = lambda res: {"class":res[4], "subclass":stablehash(res[3]), "data":[self.way_full, self.positionAsText], "text": T_f(u"{0} deviation of {1}m", res[3], res[2])}
def test(self):
a = Plugin(None)
self.assertEqual(a.init(None), None)
self.assertEqual(a.errors, {})
self.assertEqual(a.node(None, None), None)
self.assertEqual(a.way(None, None, None), None)
self.assertEqual(a.relation(None, None, None), None)
self.assertEqual(a.end(None), None)
for n in [(u"bpoue", u"bpoue"),
(u"bpoué", u"bpoue"),
(u"bpoùé", u"bpoue"),
(u"bpôùé", u"bpoue"),
]:
self.assertEqual(a.ToolsStripAccents(n[0]), n[1], n)
for n in [(u"1", u"beppu"),
(u"1", u"lhnsune"),
(u"1", u"uae"),
(u"1", u"bue"),
]:
self.assertNotEqual(stablehash(n[0]), stablehash(n[1]))
def test(self):
a = Plugin(None)
self.assertEquals(a.init(None), None)
self.assertEquals(a.errors, {})
self.assertEquals(a.node(None, None), None)
self.assertEquals(a.way(None, None, None), None)
self.assertEquals(a.relation(None, None, None), None)
self.assertEquals(a.end(None), None)
for n in [(u"bpoue", u"bpoue"),
(u"bpoué", u"bpoue"),
(u"bpoùé", u"bpoue"),
(u"bpôùé", u"bpoue"),
]:
self.assertEquals(a.ToolsStripAccents(n[0]), n[1], n)
for n in [(u"1", u"beppu"),
(u"1", u"lhnsune"),
(u"1", u"uae"),
(u"1", u"bue"),
]:
self.assertNotEqual(stablehash(n[0]), stablehash(n[1]))
def check(self, data, tags, check_list_open):
err = []
keyss = tags.keys()
keys = set(keyss) & check_list_open
for k in keys:
if not self.Values_open.match(tags[k]):
if k in self.exceptions_open:
if tags[k] in self.exceptions_open[k]:
# no error if in exception list
continue
err.append({
"class":
3040,
"subclass":
stablehash(k),
"text":
T_("Bad value for %(key)s=%(val)s", {
"key": k,
"val": tags[k]
})
})
keys = set(keyss) & self.check_list_closed
for k in keys:
if tags[k] not in self.allow_closed[k]:
err.append({
"class":
3040,
"subclass":
stablehash(k),
"text":
T_("Bad value for %(key)s=%(val)s", {
"key": k,
"val": tags[k]
})
})
return err
def node(self, data, tags):
err = []
keys = tags.keys()
for k in keys:
part = k.split(':', 1)
if ":(" in k or k.startswith("def:") or part[0] in self.exceptions:
# acess:([date])
# key def: can contains sign =
continue
if k in self.exceptions_whole:
continue
if not self.KeyPart1.match(part[0]):
err.append({
"class":
3050,
"subclass":
stablehash(k),
"text":
T_("Bad tag %(k)s=%(v)s", {
"k": k,
"v": tags[k]
})
})
elif len(part) == 2 and not self.KeyPart2.match(part[1]):
err.append({
"class":
30501,
"subclass":
stablehash(k),
"text":
T_("Bad tag suffix %(k)s=%(v)s", {
"k": k,
"v": tags[k]
})
})
return err
def node(self, data, tags):
err = []
for k in set(tags).intersection(self.DeprecatedSet):
if None in self.Deprecated[k]:
err.append({
"class":
4010,
"subclass":
stablehash(k),
"text":
T_('The tag `{0}` is deprecated in favour of {1}', k,
self.Deprecated[k][None])
})
elif tags[k] in self.Deprecated[k]:
suggestion = self.Deprecated[k][tags[k]]
fix = None
if suggestion.count(
"`"
) == 2 and "=" in suggestion and not "*" in suggestion:
# limited to suggestions with only one tag=key possibility
(sk, sv) = suggestion.split("`")[1].split("=")
if k == sk:
fix = {"~": {sk: sv}}
else:
fix = {"-": [k], "+": {sk: sv}}
err.append({
"class":
40102,
"subclass":
stablehash(k),
"text":
T_('The tag `{0}` is deprecated in favour of {1}',
"=".join([k, tags[k]]), self.Deprecated[k][tags[k]]),
"fix":
fix
})
return err
def init(self, logger):
Plugin.init(self, logger)
country = self.father.config.options.get("country") if self.father else None
language = self.father.config.options.get("language") if self.father else None
if isinstance(language, list):
language = None
self._update_ks = {}
self._update_kr = {}
self._update_ks_vs = defaultdict(dict)
self._update_kr_vs = defaultdict(dict)
self._update_ks_vr = defaultdict(dict)
self._update_kr_vr = defaultdict(dict)
reline = re.compile("^\|([^|]*)\|\|([^|]*)\|\|([^|]*)\|\|([^|]*).*")
# récupération des infos depuis https://wiki.openstreetmap.org/index.php?title=User:FrViPofm/TagwatchCleaner
data = urlread(u"https://wiki.openstreetmap.org/index.php?title=User:FrViPofm/TagwatchCleaner&action=raw", 1)
data = data.split("\n")
for line in data:
for res in reline.findall(line):
only_for = res[3].strip()
if only_for in (None, '', country, language) or (country and country.startswith(only_for)):
r = res[1].strip()
c0 = res[2].strip()
tags = ["fix:chair"] if c0 == "" else [c0, "fix:chair"]
c = stablehash(c0)
self.errors[c] = { "item": 3030, "level": 2, "tag": tags, "desc": {"en": c0} }
if u"=" in res[0]:
k = res[0].split(u"=")[0].strip()
v = res[0].split(u"=")[1].strip()
if self.quoted(k):
k = self.quoted2re(k)
if self.quoted(v):
self._update_kr_vr[k][self.quoted2re(v)] = [r, c]
else:
self._update_kr_vs[k][v] = [r, c]
else:
if self.quoted(v):
self._update_ks_vr[k][self.quoted2re(v)] = [r, c]
else:
self._update_ks_vs[k][v] = [r, c]
else:
if self.quoted(res[0]):
self._update_kr[self.quoted2re(res[0])] = [r, c]
else:
self._update_ks[res[0]] = [r, c]
def node(self, data, tags):
if u"name" not in tags:
return
if (u"highway" not in tags) and (u"waterway"
not in tags) and (u"place"
not in tags):
return
words = []
name = tags[u"name"]
name_subst = self.apply_special_subst(name)
split = self._split(name_subst)
for i in range(0, len(split), 2):
split[i] = self.remove_special_subst(split[i])
splitfix = list(split)
if split and split[0] and split[0][0] in self.minus:
words.append(split[0])
splitfix[0] = split[0].capitalize()
for i in range(0, len(split), 2):
word = split[i]
if word in self.special:
continue
if word[0] in self.minus:
words.append(word)
splitfix[i] = split[i].capitalize()
if words:
return {
"class":
906,
"subclass":
stablehash(','.join(words)),
"text":
T_(u"Missing capital letter for: %s",
u", ".join(sorted(set(words)))),
"fix": {
"name": "".join(splitfix)
}
}
return
def __init__(self, config, logger=None):
Analyser_Osmosis.__init__(self, config, logger)
self.FR = config.options and (
"country" in config.options
and config.options["country"].startswith("FR")
or "test" in config.options)
self.classs[100] = {
"item": "6070",
"level": 3,
"tag": ["boundary", "geom", "fix:chair"],
"desc": T_(u"Survey point out of boundary")
}
self.classs[2] = {
"item": "6060",
"level": 1,
"tag": ["boundary", "geom", "fix:chair"],
"desc": T_(u"Boundary intersection")
}
self.classs_change[3] = {
"item": "6060",
"level": 2,
"tag": ["boundary", "geom", "fix:chair"],
"desc": T_(u"Lone boundary fragment")
}
self.callback20 = lambda res: {
"class": 100,
"data":
[self.relation_full, self.relation_full, self.positionAsText]
}
self.callback40 = lambda res: {
"class": 2,
"subclass": stablehash(res[2]),
"data":
[self.relation_full, self.relation_full, self.positionAsText]
}
self.callback50 = lambda res: {
"class": 3,
"data": [self.way_full, self.positionAsText]
}
def _get_err(self, name):
initialName = name
err = []
for x in [u"&", u"'", u""", u"/", u")", u"-", u"\"", u";", u".", u":", u"+", u"?", u"!", u",", u"|", u"*", u"°", u"_", u"="]:
name = name.replace(x, " ")
if self.apostrophe:
name = self.apostrophe.sub(' ', name)
for WordComplet in name.split(" "):
if WordComplet in self.DictCommonWords: continue
elif WordComplet in self.DictKnownWords: continue
elif WordComplet in self.DictCorrections:
if self.DictCorrections[WordComplet]:
err.append({"class": 703, "subclass": stablehash(WordComplet), "fix": {"name": initialName.replace(WordComplet, self.DictCorrections[WordComplet])}})
else:
raise Exception("Could not find correction for %s" % WordComplet)
else:
PbEncodage = False
for x in self.DictEncoding:
if x in WordComplet:
PbEncodage = True
err.append({"class": 704, "subclass": 0, "fix": {"name": initialName.replace(x, self.DictEncoding[x])}})
if PbEncodage: continue
#if WordComplet in self.DictUnknownWords: continue
if "0" in WordComplet: continue
if "1" in WordComplet: continue
if "2" in WordComplet: continue
if "3" in WordComplet: continue
if "4" in WordComplet: continue
if "5" in WordComplet: continue
if "6" in WordComplet: continue
if "7" in WordComplet: continue
if "8" in WordComplet: continue
if "9" in WordComplet: continue
self.DictUnknownWords.add(WordComplet)
return err
def analyser_osmosis_common(self):
self.run(sql10, lambda res: {"class":1, "subclass":stablehash(res[0]), "data":[self.positionAsText]} )
def analyse(self, tags, wikipediaTag="wikipedia"):
err = []
if wikipediaTag in tags:
m = self.wiki_regexp.match(tags[wikipediaTag])
if (tags[wikipediaTag].startswith(u"http://")
or tags[wikipediaTag].startswith(u"https://")) and not m:
# tag 'wikipedia' starts with 'http://' but it's not a wikipedia url
return [{"class": 30310, "subclass": 0}]
elif m:
# tag 'wikipedia' seams to be an url
return [{
"class": 30311,
"subclass": 1,
"text": T_(u"Use wikipedia=%s:*", m.group(2)),
"fix": {
wikipediaTag:
"%s:%s" % (m.group(2), self.human_readable(m.group(3)))
}
}]
if not self.lang_regexp.match(tags[wikipediaTag]):
err.append({"class": 30312, "subclass": 2})
else:
prefix = tags[wikipediaTag].split(':', 1)[0]
tag = wikipediaTag + ':' + prefix
if tag in tags:
err.append({
"class": 30316,
"subclass": 6,
"fix": {
'-': [tag]
}
})
if "%" in tags[wikipediaTag] or "_" in tags[wikipediaTag]:
err.append({
"class": 30313,
"subclass": 3,
"fix": {
wikipediaTag: self.human_readable(tags[wikipediaTag])
}
})
interwiki = False
missing_primary = []
for tag in [t for t in tags if t.startswith(wikipediaTag + ":")]:
suffix = tag[len(wikipediaTag) + 1:]
if ":" in suffix:
suffix = suffix.split(":")[0]
if self.Country and self.Country.startswith(
"UA"
) and suffix == "ru": # In Ukraine wikipedia=uk:X + wikipedia:ru=Y are allowed
continue
if wikipediaTag in tags:
if interwiki == False:
try:
lang, title = tags[wikipediaTag].split(':')
json_str = urlread(
u"https://" + lang +
u".wikipedia.org/w/api.php?action=query&prop=langlinks&titles="
+ title + u"&redirects=&lllimit=500&format=json",
30)
interwiki = json.loads(json_str)
interwiki = dict(
map(
lambda x: [x["lang"], x["*"]],
list(interwiki["query"]["pages"].values())[0]
["langlinks"]))
except:
interwiki = None
if interwiki and suffix in interwiki and interwiki[
suffix] == self.human_readable(tags[tag]):
err.append({
"class":
30317,
"subclass":
stablehash(tag),
"fix": [{
'-': [tag]
}, {
'-': [tag],
'~': {
wikipediaTag: suffix + ':' + interwiki[suffix]
}
}]
})
if suffix in tags:
# wikipedia:xxxx only authorized if tag xxxx exist
err.extend(self.analyse(tags, wikipediaTag + ":" + suffix))
elif self.lang_restriction_regexp.match(suffix):
if not wikipediaTag in tags:
m = self.wiki_regexp.match(tags[tag])
if m:
value = self.human_readable(m.group(3))
elif tags[tag].startswith(suffix + ":"):
value = tags[tag][len(suffix) + 1:]
else:
value = self.human_readable(tags[tag])
missing_primary.append({
'-': [tag],
'+': {
wikipediaTag: "%s:%s" % (suffix, value)
}
})
else:
err.append({
"class": 30315,
"subclass": stablehash(tag),
"text": T_(u"Invalid wikipedia suffix '%s'", suffix)
})
if missing_primary != []:
if self.Language:
missing_primary = sorted(
missing_primary,
key=lambda x: x['+'][wikipediaTag][0:2]
if x['+'][wikipediaTag][0:2] != self.Language else '')
err.append({"class": 30314, "subclass": 4, "fix": missing_primary})
return err
def __init__(self, config, logger=None):
Analyser_Osmosis.__init__(self, config, logger)
self.FR = config.options and (
"country" in config.options
and config.options["country"].startswith("FR")
or "test" in config.options)
self.classs_change[1] = {
"item": "0",
"level": 3,
"tag": ["building", "geom", "fix:chair"],
"desc": T_(u"Building intersection")
}
self.classs_change[2] = {
"item": "0",
"level": 2,
"tag": ["building", "geom", "fix:chair"],
"desc": T_(u"Large building intersection")
}
self.classs_change[3] = {
"item": "0",
"level": 3,
"tag": ["building", "geom", "fix:chair"],
"desc": T_(u"Building too small")
}
self.classs_change[4] = {
"item": "0",
"level": 3,
"tag": ["building", "geom", "fix:chair"],
"desc": T_(u"Gap between buildings")
}
self.classs_change[5] = {
"item": "0",
"level": 1,
"tag": ["building", "fix:chair"],
"desc": T_(u"Large building intersection cluster")
}
if self.FR:
self.classs_change[6] = {
"item": "1",
"level": 3,
"tag": ["building", "geom", "fix:chair"],
"desc": T_(u"Building in parts")
}
self.callback30 = lambda res: {
"class": 2 if res[3] > res[4] else 1,
"data": [self.way, self.way, self.positionAsText]
}
self.callback40 = lambda res: {
"class": 3,
"data": [self.way, self.positionAsText]
}
self.callback50 = lambda res: {
"class": 4,
"data": [self.way, self.way, self.positionAsText]
}
self.callback60 = lambda res: {
"class": 5,
"subclass": stablehash(res[0]),
"data": [self.positionAsText]
}
if self.FR:
self.callback70 = lambda res: {
"class": 6,
"data": [self.way, self.positionAsText]
}
def __init__(self, config, logger=None):
Analyser_Osmosis.__init__(self, config, logger)
self.classs[1] = {
"item": "7040",
"level": 3,
"tag": ["power", "fix:imagery"],
"desc": T_(u"Lone power tower or pole")
}
self.classs[2] = {
"item": "7040",
"level": 2,
"tag": ["power", "fix:imagery"],
"desc": T_(u"Unfinished power major line")
}
self.classs[6] = {
"item": "7040",
"level": 3,
"tag": ["power", "fix:imagery"],
"desc": T_(u"Unfinished power minor line")
}
self.classs[3] = {
"item": "7040",
"level": 3,
"tag": ["power", "fix:chair"],
"desc": T_(u"Connection between different voltages")
}
self.classs_change[4] = {
"item": "7040",
"level": 3,
"tag": ["power", "fix:imagery"],
"desc": T_(u"Non power node on power way")
}
self.classs_change[5] = {
"item": "7040",
"level": 3,
"tag": ["power", "fix:imagery"],
"desc": T_(u"Missing power tower or pole")
}
self.classs[7] = {
"item": "7040",
"level": 3,
"tag": ["power", "fix:chair"],
"desc": T_(u"Unmatched voltage of line on substation")
}
self.callback40 = lambda res: {
"class": 4,
"data": [self.node_full, self.positionAsText],
"fix": [{
"+": {
"power": "tower"
}
}, {
"+": {
"power": "pole"
}
}]
}
self.callback50 = lambda res: {
"class": 5,
"subclass": stablehash(res[1]),
"data": [self.way_full, self.positionAsText]
}
def analyser_osmosis_common(self):
self.run(sql10.format("nodes"))
self.run(sql20.format("nodes"))
self.run(
sql30.format("nodes") % {
"as_text": "geom",
"table": "nodes",
"geo": "geom"
}, lambda res: {
"class":
1,
"subclass":
stablehash(res[1]),
"data":
[self.node_full, None, None, None, None, self.positionAsText],
"fix": {
"-": [res[1]],
"+": {
res[1].replace(res[3], res[4], 1): res[2]
}
}
})
self.run(sql10.format("ways"))
self.run(sql20.format("ways"))
self.run(
sql30.format("ways") % {
"as_text": "way_locate(linestring)",
"table": "ways",
"geo": "linestring"
}, lambda res: {
"class":
1,
"subclass":
stablehash(res[1]),
"data":
[self.way_full, None, None, None, None, self.positionAsText],
"fix": {
"-": [res[1]],
"+": {
res[1].replace(res[3], res[4], 1): res[2]
}
}
})
self.run(sql10.format("relations"))
self.run(sql20.format("relations"))
self.run(
sql30.format("relations") % {
"as_text": "relation_locate(id)",
"table": "relations",
"geo": "user"
}, lambda res: {
"class":
1,
"subclass":
stablehash(res[1]),
"data": [
self.relation_full, None, None, None, None, self.
positionAsText
],
"fix": {
"-": [res[1]],
"+": {
res[1].replace(res[3], res[4], 1): res[2]
}
}
})
def check(self, tags):
err = []
for tag in self.PHONE_TAGS:
if tag not in tags:
continue
phone = tags[tag]
if u';' in phone:
continue # Ignore multiple phone numbers
if self.suffix_separators is not None:
phone = phone.split(self.suffix_separators, 1)[0]
if self.values_separators:
p = phone
for sep in self.values_separators:
if sep in phone:
phone = phone.replace(sep, '; ')
if p != phone:
phone = phone.replace(' ', ' ')
err.append({"class": 30926, "subclass": stablehash(tag), "text": {'en': u'='.join([tag, phone])}, "fix": {tag: phone.replace(' / ', '; ').replace(' - ', '; ').replace(',', ';')}})
continue
phone_test = phone
for c in '+0123456789 -./()':
phone_test = phone_test.replace(c, '')
if len(phone_test) > 0:
err.append({"class": 30925, "subclass": stablehash(tag), "text": T_f(u"Not allowed char \"{0}\" in phone number tag \"{1}\"", phone_test, tag)})
continue
# Before local prefix
if self.InternationalPrefix:
r = self.InternationalPrefix.match(phone)
if r:
err.append({"class": 30924, "subclass": stablehash(tag), "text": {'en': u'='.join([tag, phone])}, "fix": {tag: "+" + r.group(1)}})
continue
if self.InternationalAndLocalPrefix:
r = self.InternationalAndLocalPrefix.match(phone)
if r:
err.append({"class": 30921, "subclass": stablehash(tag), "text": {'en': u'='.join([tag, phone])}, "fix": {tag: "+" + self.code + " " + r.group(1)}})
continue
if self.MissingInternationalPrefix:
r = self.MissingInternationalPrefix.match(phone)
if r:
err.append({"class": 30923, "subclass": stablehash(tag), "text": {'en': u'='.join([tag, phone])}, "fix": {tag: "+" + self.code + " " + r.group(1)}})
continue
if self.BadShort:
r = self.BadShort.match(phone)
if r:
err.append({"class": 30922, "subclass": stablehash(tag), "text": {'en': u'='.join([tag, phone])}, "fix": {tag: r.group(1)}})
continue
# Last
if self.Format:
r = self.Format.match(phone)
if not r:
err.append({"class": 30920, "subclass": stablehash(tag), "text": {'en': u'='.join([tag, phone])}})
continue
return err
def analyser_osmosis_common(self):
self.run(
"SET search_path TO %s" % (self.config.db_schema_path or ','.join(
[self.config.db_user, self.config.db_schema, 'public']), ))
table = self.load.run(self, self.parser, self.mapping,
self.config.db_user,
self.__class__.__name__.lower()[15:],
self.analyser_version())
if not table:
self.logger.log(u"Empty bbox, abort")
return
# Extract OSM objects
if self.load.srid:
typeSelect = {
'N': 'geom',
'W': 'linestring',
'R': 'relation_locate(id)'
}
typeGeom = {
'N': 'geom',
'W': 'way_locate(linestring)',
'R': 'relation_locate(id)'
}
if self.mapping.osmRef == "NULL" or self.possible_merge:
typeShape = {
'N': 'geom',
'W': 'ST_Envelope(linestring)',
'R': 'relation_shape(id)'
}
else:
typeShape = {'N': 'NULL', 'W': 'NULL', 'R': 'NULL'}
else:
typeSelect = {'N': 'NULL', 'W': 'NULL', 'R': 'NULL'}
typeGeom = {'N': 'NULL', 'W': 'NULL', 'R': 'NULL'}
typeShape = {'N': 'NULL', 'W': 'NULL', 'R': 'NULL'}
self.logger.log(u"Retrive OSM item")
where = "(" + (") OR (".join(
map(lambda x: self.where(x), self.mapping.select.tags))) + ")"
self.run("CREATE TEMP TABLE osm_item AS " + ("UNION".join(
map(
lambda type: ("""(
SELECT
'%(type)s'::char(1) AS type,
id,
trim(both from ref) AS ref,
%(geom)s::geography AS geom,
%(shape)s::geography AS shape,
tags
FROM
%(from)s
LEFT JOIN LATERAL regexp_split_to_table(tags->'%(ref)s', ';') a(ref) ON true
WHERE""" + ("""
%(geomSelect)s IS NOT NULL AND""" if self.load.srid
else "") + ("""
ST_SetSRID(ST_GeomFromText('%(bbox)s'), 4326) && %(geomSelect)s AND"""
if self.load.bbox and self.load.
srid else "") + """
tags != ''::hstore AND
%(where)s)""") %
{
"type": type[0].upper(),
"ref": self.mapping.osmRef,
"geomSelect": typeSelect[type[0].upper()],
"geom": typeGeom[type[0].upper()],
"shape": typeShape[type[0].upper()],
"from": type,
"bbox": self.load.bbox,
"where": where
}, self.mapping.select.types))))
if self.mapping.osmRef != "NULL":
self.run("CREATE INDEX osm_item_index_ref ON osm_item(ref)")
self.run(
"CREATE INDEX osm_item_index_shape ON osm_item USING GIST(shape)")
joinClause = []
if self.mapping.osmRef != "NULL":
joinClause.append("official.ref = osm_item.ref")
elif self.load.srid:
joinClause.append("ST_DWithin(official.geom, osm_item.shape, %s)" %
self.mapping.conflationDistance)
if self.mapping.extraJoin:
joinClause.append(
"official.tags->'%(tag)s' = osm_item.tags->'%(tag)s'" %
{"tag": self.mapping.extraJoin})
joinClause = " AND\n".join(joinClause) + "\n"
# Missing official
self.run(sql10 % {"official": table, "joinClause": joinClause})
self.run(sql11)
if self.missing_official:
self.run(
sql12, lambda res: {
"class":
self.missing_official["class"],
"subclass":
str(stablehash("%s%s" % (res[0], res[1]))),
"self":
lambda r: [0] + r[1:],
"data": [self.node_new, self.positionAsText],
"text":
self.mapping.generate.text(
defaultdict(lambda: None, res[2]),
defaultdict(lambda: None, res[3])),
"fix":
self.passTags(res[2]) if self.mapping.generate.
missing_official_fix and res[2] != {} else None,
})
if self.mapping.osmRef != "NULL":
self.run(sql20 % {"official": table, "joinClause": joinClause})
self.run(sql21)
if self.missing_osm:
# Missing OSM
self.run(
sql22, lambda res: {
"class":
self.missing_osm["class"],
"data":
[self.typeMapping[res[1]], None, self.positionAsText]
})
# Invalid OSM
self.run(
sql23 % {
"official": table,
"joinClause": joinClause
}, lambda res: {
"class":
self.missing_osm["class"],
"data":
[self.typeMapping[res[1]], None, self.positionAsText]
})
# Possible merge
if self.possible_merge:
possible_merge_joinClause = []
possible_merge_orderBy = ""
if self.load.srid:
possible_merge_joinClause.append(
"ST_DWithin(missing_official.geom, missing_osm.shape, %s)"
% self.mapping.conflationDistance)
possible_merge_orderBy = ", ST_Distance(missing_official.geom, missing_osm.shape) ASC"
if self.mapping.extraJoin:
possible_merge_joinClause.append(
"missing_official.tags->'%(tag)s' = missing_osm.tags->'%(tag)s'"
% {"tag": self.mapping.extraJoin})
possible_merge_joinClause = " AND\n".join(
possible_merge_joinClause) + "\n"
self.run(
sql30 % {
"joinClause": possible_merge_joinClause,
"orderBy": possible_merge_orderBy
}, lambda res: {
"class":
self.possible_merge["class"],
"subclass":
str(stablehash("%s%s" % (res[0], str(res[3])))),
"data":
[self.typeMapping[res[1]], None, self.positionAsText],
"text":
self.mapping.generate.text(
defaultdict(lambda: None, res[3]),
defaultdict(lambda: None, res[4])),
"fix":
self.mergeTags(
res[5], res[3], self.mapping.osmRef, self.mapping.
generate.tag_keep_multiple_values),
})
self.dumpCSV(
"SELECT ST_X(geom::geometry) AS lon, ST_Y(geom::geometry) AS lat, tags FROM %s"
% table, "", ["lon", "lat"], lambda r, cc: list(
(r['lon'], r['lat'])) + cc)
self.run(sql40 % {"official": table, "joinClause": joinClause})
self.dumpCSV(
sql41, ".byOSM", ["osm_id", "osm_type", "lon", "lat"],
lambda r, cc: list(
(r['osm_id'], r['osm_type'], r['lon'], r['lat'])) + cc)
file = io.open("%s/%s.metainfo.csv" %
(self.config.dst_dir, self.name),
"w",
encoding="utf8")
file.write(
u"file,origin,osm_date,official_non_merged,osm_non_merged,merged\n"
)
if self.missing_official:
self.giscurs.execute("SELECT COUNT(*) FROM missing_official;")
official_non_merged = self.giscurs.fetchone()[0]
else:
official_non_merged = 0
self.giscurs.execute("SELECT COUNT(*) FROM missing_osm;")
osm_non_merged = self.giscurs.fetchone()[0]
self.giscurs.execute("SELECT COUNT(*) FROM match;")
merged = self.giscurs.fetchone()[0]
file.write(u"\"%s\",\"%s\",FIXME,%s,%s,%s\n" %
(self.name, self.parser.source.fileUrl or self.url,
official_non_merged, osm_non_merged, merged))
file.close()
# Moved official
if self.moved_official:
self.run(
sql50 % {
"official": table,
"joinClause": joinClause
}, lambda res: {
"class": self.moved_official["class"],
"data": [self.node_full, self.positionAsText],
})
# Update official
if self.update_official:
self.run(
sql60 % {
"official": table,
"joinClause": joinClause
}, lambda res: {
"class":
self.update_official["class"],
"subclass":
str(stablehash("%s%s" % (res[0], str(res[4])))),
"data":
[self.typeMapping[res[1]], None, self.positionAsText],
"text":
self.mapping.generate.text(
defaultdict(lambda: None, res[3]),
defaultdict(lambda: None, res[5])),
"fix":
self.mergeTags(
res[4], res[3], self.mapping.osmRef, self.mapping.
generate.tag_keep_multiple_values),
})
def node(self, data, tags):
err = []
for k in tags:
if k in self._update_ks:
err.append({
"class":
self._update_ks[k][1],
"subclass":
stablehash(k),
"text":
T_f(u"tag key: {0} => {1} (rule ks)", k,
self._update_ks[k][0])
})
if k in self._update_ks_vs and tags[k] in self._update_ks_vs[k]:
err.append({
"class":
self._update_ks_vs[k][tags[k]][1],
"subclass":
stablehash(u"%s=%s" % (k, tags[k])),
"text":
T_f(u"tag value: {0}={1} => {2} (rule ks_vs)", k, tags[k],
self._update_ks_vs[k][tags[k]][0])
})
if k in self._update_ks_vr:
for v in self._update_ks_vr[k]:
if v.match(tags[k]):
err.append({
"class":
self._update_ks_vr[k][v][1],
"subclass":
stablehash(u"%s=%s" % (k, tags[k])),
"text":
T_f(u"tag value: {0}={1} => {2} (rule ks_vr)", k,
tags[k], self._update_ks_vr[k][v][0])
})
for kk in tags:
for k in self._update_kr:
if k.match(kk):
err.append({
"class":
self._update_kr[k][1],
"subclass":
stablehash(kk),
"text":
T_f(u"tag key: {0} => {1} (rule kr)", kk,
self._update_kr[k][0])
})
for k in self._update_kr_vs:
if k.match(kk):
if tags[kk] in self._update_kr_vs[k]:
err.append({
"class":
self._update_kr_vs[k][tags[kk]][1],
"subclass":
stablehash(u"%s=%s" % (kk, tags[kk])),
"text":
T_f(u"tag value: {0}={1} => {2} (rule kr_vs)", kk,
tags[kk], self._update_kr_vs[k][tags[kk]][0])
})
for k in self._update_kr_vr:
if k.match(kk):
for v in self._update_kr_vr[k]:
if v.match(tags[kk]):
err.append({
"class":
self._update_kr_vr[k][v][1],
"zsubclass":
stablehash(u"%s=%s" % (kk, tags[kk])),
"text":
T_f(u"tag value: {0}={1} => {2} (rule ks_vr)",
kk, tags[kk], self._update_kr_vr[k][v][0])
})
return err
def init(self, logger):
Plugin.init(self, logger)
country = self.father.config.options.get(
"country") if self.father else None
language = self.father.config.options.get(
"language") if self.father else None
if isinstance(language, list):
language = None
self._update_ks = {}
self._update_kr = {}
self._update_ks_vs = defaultdict(dict)
self._update_kr_vs = defaultdict(dict)
self._update_ks_vr = defaultdict(dict)
self._update_kr_vr = defaultdict(dict)
reline = re.compile("^\|([^|]*)\|\|([^|]*)\|\|([^|]*)\|\|([^|]*).*")
# récupération des infos depuis https://wiki.openstreetmap.org/index.php?title=User:FrViPofm/TagwatchCleaner
data = urlread(
u"https://wiki.openstreetmap.org/index.php?title=User:FrViPofm/TagwatchCleaner&action=raw",
1)
data = data.split("\n")
for line in data:
for res in reline.findall(line):
only_for = res[3].strip()
if only_for in (None, '', country, language) or (
country and country.startswith(only_for)):
r = res[1].strip()
c0 = res[2].strip()
tags = ["fix:chair"] if c0 == "" else [c0, "fix:chair"]
c = stablehash(c0)
self.errors[c] = {
"item": 3030,
"level": 2,
"tag": tags,
"desc": {
"en": c0
}
}
if u"=" in res[0]:
k = res[0].split(u"=")[0].strip()
v = res[0].split(u"=")[1].strip()
if self.quoted(k):
k = self.quoted2re(k)
if self.quoted(v):
self._update_kr_vr[k][self.quoted2re(v)] = [
r, c
]
else:
self._update_kr_vs[k][v] = [r, c]
else:
if self.quoted(v):
self._update_ks_vr[k][self.quoted2re(v)] = [
r, c
]
else:
self._update_ks_vs[k][v] = [r, c]
else:
if self.quoted(res[0]):
self._update_kr[self.quoted2re(res[0])] = [r, c]
else:
self._update_ks[res[0]] = [r, c]
def node(self, data, tags):
err = []
for key, value in tags.items():
m = self.non_printable.search(key)
if m:
err.append({"class": 50702, "subclass": 0 + stablehash(key), "text": T_f(u"\"{0}\" unexpected non printable char ({1}, 0x{2:04x}) in key at position {3}", key, unicodedata.name(m.group(0), ''), ord(m.group(0)), m.start() + 1)})
break
m = self.non_printable.search(value)
if m:
err.append({"class": 50702, "subclass": 1 + stablehash(key), "text": T_f(u"\"{0}\"=\"{1}\" unexpected non printable char ({2}, 0x{3:04x}) in value at position {4}", key, value, unicodedata.name(m.group(0), ''), ord(m.group(0)), m.start() + 1)})
break
m = self.other_symbol.search(key)
if m:
err.append({"class": 50703, "subclass": 0 + stablehash(key), "text": T_f(u"\"{0}\" unexpected symbol char ({1}, 0x{2:04x}) in key at position {3}", key, unicodedata.name(m.group(0), ''), ord(m.group(0)), m.start() + 1)})
break
m = self.other_symbol.search(value)
if m:
err.append({"class": 50703, "subclass": 1 + stablehash(key), "text": T_f(u"\"{0}\"=\"{1}\" unexpected symbol char ({2}, 0x{3:04x}) in value at position {4}", key, value, unicodedata.name(m.group(0), ''), ord(m.group(0)), m.start() + 1)})
break
err_size = len(err)
# https://en.wikipedia.org/wiki/Bi-directional_text#Table_of_possible_BiDi-types
for c in u"\u200E\u200F\u061C\u202A\u202D\u202B\u202E\u202C\u2066\u2067\u2068\u2069":
m = key.find(c)
if m > 0:
err.append({"class": 50702, "subclass": 2 + stablehash(key), "text": T_f(u"\"{0}\" unexpected non printable char ({1}, 0x{2:04x}) in key at position {3}", key, unicodedata.name(c, ''), ord(c), m + 1)})
break
m = value.find(c)
if m > 0:
err.append({"class": 50702, "subclass": 3 + stablehash(key), "text": T_f(u"\"{0}\"=\"{1}\" unexpected non printable char ({2}, 0x{3:04x}) in value at position {4}", key, value, unicodedata.name(c, ''), ord(c), m + 1)})
break
if err_size != len(err):
break
if self.default:
if key in self.names:
s = self.non_letter.sub(u" ", value)
s = self.alone_char.sub(u"", s)
s = self.roman_number.sub(u"", s)
s = self.default.sub(u"", s)
if len(s) > 0 and not(len(value) == 2 and len(s) == 1) and len(s) <= len(value) / 10 + 1:
if len(s) == 1:
c = s[0]
u = self.uniq_script and confusables.unconfuse(c, self.uniq_script)
if u:
err.append({"class": 50701, "subclass": 0 + stablehash(key),
"text": T_f(u"\"{0}\"=\"{1}\" unexpected char \"{2}\" ({3}, 0x{4:04x}). Means \"{5}\" ({6}, 0x{7:04x})?", key, value, s, unicodedata.name(c, ''), ord(c), u, unicodedata.name(u, ''), ord(u)),
"fix": {key: value.replace(c, u)}
})
else:
err.append({"class": 50701, "subclass": 0 + stablehash(key),
"text": T_f(u"\"{0}\"=\"{1}\" unexpected char \"{2}\" ({3}, 0x{4:04x})", key, value, s, unicodedata.name(c, ''), ord(c))
})
else:
err.append({"class": 50701, "subclass": 0 + stablehash(key), "text": T_f(u"\"{0}\"=\"{1}\" unexpected \"{2}\"", key, value, s)})
break
l = key.split(':')
if len(l) > 1 and l[0] in self.names and l[1] in self.lang:
s = self.non_letter.sub(u" ", value)
s = self.alone_char.sub(u"\\1", s)
s = self.roman_number.sub(u"\\1", s)
s = self.lang[l[1]].sub(u"", s)
if len(s) > 0:
if len(s) == 1:
c = s[0]
u = self.uniq_scripts.get(l[1]) and confusables.unconfuse(c, self.uniq_scripts.get(l[1]))
if u:
err.append({"class": 50701, "subclass": 1 + stablehash(key),
"text": T_f(u"\"{0}\"=\"{1}\" unexpected char \"{2}\" ({3}, 0x{4:04x}). Means \"{5}\" ({6}, 0x{7:04x})?", key, value, s, unicodedata.name(c, ''), ord(c), u, unicodedata.name(u, ''), ord(u)),
"fix": {key: value.replace(c, u)}
})
else:
err.append({"class": 50701, "subclass": 1 + stablehash(key),
"text": T_f(u"\"{0}\"=\"{1}\" unexpected char \"{2}\" ({3}, 0x{4:04x})", key, value, s, unicodedata.name(c, ''), ord(c))
})
else:
err.append({"class": 50701, "subclass": 1 + stablehash(key), "text": T_f(u"\"{0}\"=\"{1}\" unexpected \"{2}\"", key, value, s)})
break
return err
def way(self, data, tags, nds):
if not "highway" in tags:
return
lanes = False
for tag in tags:
if "lanes" in tag:
lanes = True
break
if not lanes:
return
tags_lanes = {}
for tag in tags:
if "lanes" in tag and not "source" in tag and not "FIXME" in tag:
tags_lanes[tag] = tags[tag]
err = []
# Check trun lanes values
tl = "turn:lanes" in tags_lanes
tlf = "turn:lanes:forward" in tags_lanes
tlb = "turn:lanes:backward" in tags_lanes
tl2 = "turn:lanes:both_ways" in tags_lanes
if tl or tlf or tlb or tl2:
for tl in ["turn:lanes", "turn:lanes:forward", "turn:lanes:backward", "turn:lanes:both_ways"]:
if tl in tags_lanes:
ttt = tags_lanes[tl].split("|")
unknown = False
i = 0
for tt in ttt:
for t in set(tt.split(";")):
if t not in ["left", "slight_left", "sharp_left", "through", "right", "slight_right", "sharp_right", "reverse", "merge_to_left", "merge_to_right", "none", ""]:
unknown = True
err.append({"class": 31606, "subclass": 0 + stablehash(tl + '|' + tt + '|' + str(i)), "text": T_f(u"Unknown turn lanes value \"{0}\"", t)})
if (t == "merge_to_left" and i == 0) or (t == "merge_to_right" and i == len(ttt) - 1):
err.append({"class": 31600, "subclass": 1 + stablehash(tl + '|' + tt + '|' + str(i))})
i += 1
if not unknown:
# merge_to_left is a on the right and vice versa
t = tags_lanes[tl] \
.replace("left", "l").replace("slight_left", "l").replace("sharp_left", "l") \
.replace("through", " ") \
.replace("right", "r").replace("slight_right", "r").replace("sharp_right", "r") \
.replace("reverse", "U") \
.replace("merge_to_left", "r").replace("merge_to_right", "l") \
.replace("none", " ").replace(";", "").split("|")
t = ''.join(map(lambda e: " " if len(e) == 0 or e[0] != e[-1] else e[0], map(sorted, t)))
t = t.replace('U', '') # Ignore reverse
last_left = self.rindex_(t, "l")
first_space = self.index_(t, " ")
last_space = self.rindex_(t, " ")
first_right = self.index_(t, "r")
# Check right is on the right and left is on the left...
if not(
(last_left == None or first_space == None or last_left < first_space) and
(first_space == None or last_space == None or first_space <= last_space) and
(last_space == None or first_right == None or last_space < first_right) and
(last_left == None or first_right == None or last_left < first_right)):
err.append({"class": 31607, "subclass": 1 + stablehash(tl)})
# Check acces lanes values
# Count for non fullwidth lanes
non_fullwidth_lanes_number = {}
for direction in ['', ':forward', ':backward', ':both_ways']:
o = tags_lanes.get('bicycle:lanes'+direction)
if o:
non_fullwidth_lanes_number[direction] = ilen(filter(lambda i: i == 'designated', o.split('|')))
for access in ['hgv', 'bus', 'access', 'bicycle', 'psv', 'taxi', 'vehicle', 'motor_vehicle', 'hov', 'motorcycle', 'goods']:
base = access+':lanes'
for tag in tags_lanes:
if tag.startswith(base):
try:
int(tags_lanes[tag])
err.append({"class": 31609, "subclass": 1 + stablehash(tag), "text": {'en': '%s=%s' % (tag, tags_lanes[tag]) }})
except ValueError:
# Ok, should not be an integer
pass
stars = []
for tag in tags_lanes:
if ":lanes" in tag:
star = tag.split(':')[0]
if star not in ('source', 'proposed', 'construction', 'note'):
stars.append(star)
stars = list(set(stars))
for star in stars:
l = star + '' in tags_lanes
lf = star + ':forward' in tags_lanes
lb = star + ':backward' in tags_lanes
l2 = star + ':both_ways' in tags_lanes
if l and (lf or lb or l2):
err.append({"class": 31603, "subclass": 0 + stablehash(star), "text": {"en": star + ":*"}})
if err != []:
return err
number = {'lanes': {}}
for tag in tags_lanes:
if tag == 'lanes' or tag.startswith('lanes:'):
try:
n = int(tags_lanes[tag])
parts = tag.split(':')
direction = ''
if len(parts) == 1:
number['lanes'][''] = n
elif len(parts) == 2 and parts[1] in ['forward', 'backward', 'both_ways']:
number['lanes'][':'+parts[1]] = n
except ValueError:
err.append({"class": 31601, "subclass": 0 + stablehash(tag), "text": T_f(u"lanes={0} is not an integer", tags_lanes[tag])})
for star in stars:
number[star] = {}
for direction in ['', ':forward', ':backward', ':both_ways']:
o = tags_lanes.get(star+':lanes'+direction)
if o:
number[star][direction] = len(o.split('|'))
n_lanes = {}
for direction in ['', ':forward', ':backward', ':both_ways']:
tag = None
for star in sorted(number.keys()):
non_fullwidth_lanes_number_star = ((non_fullwidth_lanes_number.get(direction) or 0) if star != 'lanes' else 0)
non_fullwidth_lanes_number_tag = ((non_fullwidth_lanes_number.get(direction) or 0) if tag != 'lanes:lanes'+direction else 0)
if n_lanes.get(direction) != None and number[star].get(direction) != None and \
number[star][direction] - non_fullwidth_lanes_number_star != \
n_lanes[direction] - non_fullwidth_lanes_number_tag:
err.append({"class": 31608, "subclass": 0 + stablehash(direction + '|' + star), "text": {
"en": "(lanes(%s)=%s) - (non fullwidth=%s) != (lanes(%s)=%s) - (non fullwidth=%s)" % (
star+":*"+direction, number[star][direction], non_fullwidth_lanes_number_star,
tag, n_lanes[direction], non_fullwidth_lanes_number_tag) }})
elif n_lanes.get(direction) == None and number[star].get(direction) != None:
# Fist loop, pick the star as tag and the number of lanes to compare to the others
n_lanes[direction] = number[star][direction]
tag = star+":lanes"+direction
if err != []:
return err
if tags["highway"] == 'motorway':
oneway = "oneway" not in tags or tags["oneway"] not in ["no", "false"]
else:
oneway = "oneway" in tags and tags["oneway"] not in ["no", "false"]
if oneway:
for tag in tags:
if tag.startswith('oneway:') and tags[tag] in ["no", "false"]:
# Oneway for mainstream traffic, but not for an other one, so we are not really on a oneway
oneway = False
if tags.get('junction') == 'roundabout':
oneway = True
nl = n_lanes.get('')
nlf = n_lanes.get(':forward')
nlb = n_lanes.get(':backward')
nl2 = n_lanes.get(':both_ways')
nfw_nl = non_fullwidth_lanes_number.get('') or 0
nfw_nlf = non_fullwidth_lanes_number.get(':forward') or 0
nfw_nlb = non_fullwidth_lanes_number.get(':backward') or 0
nfw_nl2 = non_fullwidth_lanes_number.get(':both_ways') or 0
if oneway:
if nl != None and nlf != None and nl != nlf - nfw_nlf:
err.append({"class": 31604, "subclass": 0, "text": T_f(u"on oneway, (lanes={0}) != (lanes:forward={1}) - (non fullwidth forward={2})", nl, nlf, nfw_nlf)})
elif nlb != None or nl2 != None:
err.append({"class": 31605, "subclass": 0})
else:
if nl != None and nlf != None and nlb != None and nl != nlf + nlb + (nl2 or 0) - nfw_nl - nfw_nlf - nfw_nlb - nfw_nl2:
err.append({"class": 31604, "subclass": 0, "text": T_f(u"on two way, (lanes={0}) != (lanes:forward={1}) + (lanes:backward={2}) + (lanes:both_ways={3}) - (non fullwidth={4}) - (non fullwidth forward={5}) - (non fullwidth backward={6}) - (non fullwidth both_ways={7})", nl, nlf, nlb, nl2, nfw_nl, nfw_nlf, nfw_nlb, nfw_nl2)})
elif nl != None and nlf != None and nl <= nlf - nfw_nlf:
err.append({"class": 31604, "subclass": 0, "text": T_f(u"on two way, (lanes={0}) <= (lanes:forward={1}) - (non fullwidth forward={2})", nl, nlf, nfw_nlf)})
elif nl != None and nlb != None and nl <= nlb - nfw_nlb:
err.append({"class": 31604, "subclass": 0, "text": T_f(u"on two way, (lanes={0}) <= (lanes:backward={1}) - (non fullwidth backward={2})", nl, nlb, nfw_nlb)})
elif nl != None and nl2 != None and nl < nl2 - nfw_nl2:
err.append({"class": 31604, "subclass": 0, "text": T_f(u"on two way, (lanes={0}) < (lanes:both_ways={1}) - (non fullwidth both_ways={2})", nl, nl2, nfw_nl2)})
if err != []:
return err
