def analyser(self):
apiconn = OsmGis.OsmGis(self.config.db_string, self.config.db_schema)
## result file
outxml = OsmSax.OsmSaxWriter(open(self.config.dst, "w"), "UTF-8")
outxml.startDocument()
outxml.startElement(
"analyser",
{"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())})
outxml.startElement("class", {"id": "1", "item": "0"})
outxml.Element("classtext", {
"lang": "en",
"title": "Building intersection"
})
outxml.endElement("class")
## gis connection
gisconn = psycopg2.connect(self.config.db_string)
giscurs = gisconn.cursor()
## sql querries
sql1 = "CREATE TEMP TABLE %s_building AS SELECT osm_id, way FROM %s_polygon WHERE st_isvalid(way)='t' AND st_issimple(way)='t' AND building='yes';" % (
self.config.db_schema, self.config.dbp)
sql2 = "CREATE INDEX %s_building_idx ON %s_building USING gist(way);" % (
self.config.db_schema, self.config.dbp)
sql3 = "SELECT b1.osm_id AS id1, b2.osm_id AS id2, astext(st_transform(st_pointonsurface(ST_Intersection(b1.way, b2.way)), 4020)) FROM %s_building AS b1, %s_building AS b2 WHERE b1.osm_id>b2.osm_id AND st_intersects(b1.way, b2.way)='t' AND st_area(ST_Intersection(b1.way, b2.way))<>0;" % (
self.config.db_schema, self.config.dbp)
sql4 = "DROP TABLE %s_building;" % (self.config.db_schema)
## gis querries
self.logger.log(u"create building table")
giscurs.execute(sql1)
self.logger.log(u"create building index")
giscurs.execute(sql2)
self.logger.log(u"analyse overlap")
giscurs.execute(sql3)
## format results to outxml
self.logger.log(u"generate xml")
while True:
many = giscurs.fetchmany(1000)
if not many:
break
for res in many:
outxml.startElement("error", {"class": "1"})
for loc in self.get_points(res[2]):
outxml.Element("location", loc)
#outxml.WayCreate(apiconn.WayGet(res[0]))
#outxml.WayCreate(apiconn.WayGet(res[1]))
outxml.WayCreate({"id": res[0], "nd": [], "tag": {}})
outxml.WayCreate({"id": res[1], "nd": [], "tag": {}})
outxml.endElement("error")
giscurs.close()
outxml.endElement("analyser")
outxml._out.close()
def analyser(self):
## result file
outxml = OsmSax.OsmSaxWriter(open(self.config.dst, "w"), "UTF-8")
outxml.startDocument()
outxml.startElement(
"analyser",
{"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())})
outxml.startElement("class", {"id": "1", "item": "0"})
outxml.Element(
"classtext", {
"lang": "fr",
"title": "TEST : Plusieurs nœuds à la même position",
"menu": "test : plusieurs nœuds"
})
outxml.endElement("class")
## loop on results
conn = psycopg2.connect(self.config.db_string)
curs = conn.cursor()
curs.execute(
"SELECT node_accum(id,tags),lat,lon FROM france_nodes GROUP BY lat,lon HAVING count(*) <> 1;"
)
while True:
all = curs.fetchmany(1000)
if not all:
break
for res in all:
outxml.startElement("error", {"class": "1"})
outxml.Element(
"location", {
"lat": str(float(res["lat"]) / 1000000),
"lon": str(float(res["lon"]) / 1000000)
})
for i in range(len(res[0]) / 2):
if res[0][2 * i] == "#new#":
if i:
outxml.endElement("node")
outxml.startElement("node", {"id": res[0][2 * i + 1]})
else:
outxml.Element("tag", {
"k": res[0][2 * i],
"v": res[0][2 * i + 1]
})
outxml.endElement("node")
outxml.endElement("error")
outxml.endElement("analyser")
outxml._out.close()
## update front-end
#if config.updt:
# logger.log("update front-end")
# urllib.urlretrieve(config.updt, "/dev/null")
## close database connections
curs.close()
conn.close()
def _fix(version, db, uuid, fix_num, fix):
if fix:
response.content_type = 'text/xml; charset=utf-8'
for res in fix:
if 'id' in res and res['id']:
out = io.StringIO()
o = OsmSaxFixWriter(out, "UTF-8", res['type'], res['id'],
res['create'], res['modify'],
res['delete'])
o.startDocument()
data_type = {"N": "node", "W": "way", "R": "relation"}
osm_read = utils.fetch_osm_data(data_type[res['type']],
res['id'])
osm_read.CopyTo(o)
return out.getvalue()
else:
# create new object
data = {}
data["id"] = -1
data["tag"] = {}
for (k, v) in res['create'].items():
data["tag"][k] = v
if version == 2:
sql = "SELECT lat, lon FROM markers WHERE id = %s"
db.execute(sql, (err_id, ))
else:
sql = "SELECT lat, lon FROM markers WHERE uuid = %s"
db.execute(sql, (uuid, ))
res2 = db.fetchone()
data["lat"] = res2["lat"]
data["lon"] = res2["lon"]
data[
"action"] = "modify" # Even for creation action is 'modify'
if 'type' not in res or res['type'] == 'N':
return OsmSax.NodeToXml(data, full=True)
elif res['type'] == 'W':
return OsmSax.WayToXml(data, full=True)
elif res['type'] == 'R':
return OsmSax.RelationToXml(data, full=True)
else:
abort(412, "Precondition Failed")
def analyser(self):
apiconn = OsmGis.OsmGis(self.config.db_string, self.config.db_schema)
## result file
outxml = OsmSax.OsmSaxWriter(open(self.config.dst, "w"), "UTF-8")
outxml.startDocument()
outxml.startElement(
"analyser",
{"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())})
outxml.startElement("class", {"id": "1", "item": "1050"})
outxml.Element(
"classtext", {
"lang": "fr",
"title": "Rond-point à l'envers",
"menu": "rond-point à l'envers"
})
outxml.Element(
"classtext", {
"lang": "en",
"title": "Reverse roundabout",
"menu": "reverse roundabout"
})
outxml.endElement("class")
## sql querry
sql = """
select osm_id, astext(st_transform(st_centroid(way),4020)) as center from %s_line
where junction = 'roundabout'
and (st_isclosed(way) and st_isring(way))
and ((st_azimuth(st_centroid(way),ST_PointN(way,1))-st_azimuth(st_centroid(way),ST_PointN(way,2)))::numeric)%%((pi()*2)::numeric) between pi() and pi()*2
;
""" % self.config.db_schema
gisconn = psycopg2.connect(self.config.db_string)
giscurs = gisconn.cursor()
giscurs.execute(sql)
## format results to outxml
for res in giscurs.fetchall():
outxml.startElement("error", {"class": "1"})
for loc in self.get_points(res[1]):
outxml.Element("location", loc)
#outxml.Element("text", {"lang":"en", "value":get_error_text(res[0])})
if res[0] < 0:
outxml.RelationCreate(apiconn.RelationGet(-res[0]))
else:
outxml.WayCreate(apiconn.WayGet(res[0]))
outxml.endElement("error")
outxml.endElement("analyser")
outxml._out.close()
def _osm_changeset(tags, id='0'):
out = io.StringIO()
o = OsmSax.OsmSaxWriter(out, "UTF-8")
o.startDocument()
o.startElement('osm', {"version": "0.6", "generator": u"Osmose"})
o.startElement('changeset', {"id": id, "open": "false"})
for (k, v) in tags.items():
o.Element('tag', {'k': k, 'v': v})
o.endElement('changeset')
o.endElement('osm')
return out.getvalue()
def osc_modif(config, options):
if options.poly:
from modules import OsmGeom
f = open(options.poly, "r")
name = f.readline().strip()
poly = OsmGeom.read_multipolygon(f)
poly_buffered = poly.buffer(0.1, 8)
f.close()
else:
poly = None
try:
from modules.OsmBin import OsmBin
if not hasattr(options, "osmbin_path"):
options.osmbin_path = "/data/work/osmbin/data/"
reader = OsmBin(options.osmbin_path)
except IOError:
from modules import OsmOsis
reader = OsmOsis.OsmOsis(config.dbs, config.dbp)
in_osc = OsmSax.OscSaxReader(options.source)
if options.position_only:
out_osc = OsmSax.OscPositionSaxWriter(options.dest, "UTF-8", reader)
elif poly:
out_osc = OsmSax.OscFilterSaxWriter(options.dest, "UTF-8", reader,
OsmGeom.check_intersection, poly,
poly_buffered)
elif options.bbox:
out_osc = OsmSax.OscBBoxSaxWriter(options.dest, "UTF-8", reader)
else:
# out_osc = OsmSax.OscSaxWriter(options.dest, "UTF-8", reader)
out_osc = OsmSax.OscSaxWriter(options.dest, "UTF-8")
in_osc.CopyTo(out_osc)
del in_osc
del out_osc
del reader
def analyser(self):
apiconn = OsmGis.OsmGis(self.config.db_string, self.config.db_schema)
## result file
outxml = OsmSax.OsmSaxWriter(open(self.config.dst, "w"), "UTF-8")
outxml.startDocument()
outxml.startElement("analyser", {"timestamp":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())})
outxml.startElement("class", {"id":"1", "item":"1060"})
outxml.Element("classtext", {"lang":"fr", "title":"Croisement de frontières", "menu":"boundary intersect"})
outxml.Element("classtext", {"lang":"en", "title":"Boundary intersection", "menu":"boundary intersect"})
outxml.endElement("class")
## sql querry
sql = """
select ligne1.osm_id, ligne2.osm_id, astext(st_transform(st_intersection(ligne1.way, ligne2.way),4020))
from %s_roads as ligne1, %s_roads as ligne2
where st_crosses(ligne1.way,ligne2.way) = 't'
and ST_touches(ligne1.way,ligne2.way) = 'f'
and ligne1.boundary='administrative'
and ligne2.boundary='administrative'
and ligne1.osm_id > 0
and ligne2.osm_id > 0
and ligne1.osm_id > ligne2.osm_id
;
""" % (self.config.db_schema, self.config.dbp)
gisconn = psycopg2.connect(self.config.db_string)
giscurs = gisconn.cursor()
giscurs.execute(sql)
## format results to outxml
while True:
many = giscurs.fetchmany(1000)
if not many:
break
for res in many:
outxml.startElement("error", {"class":"1"})
for loc in self.get_points(res[2]):
outxml.Element("location", loc)
outxml.WayCreate(apiconn.WayGet(res[0]))
outxml.WayCreate(apiconn.WayGet(res[1]))
outxml.endElement("error")
outxml.endElement("analyser")
outxml._out.close()
def update(wanted_end_sequence=None):
global pool
global pool_jobs
global lock_num_launched
global num_launched
# get lock
if not os.path.exists(work_path):
os.makedirs(work_path)
lock = lockfile.FileLock(lock_file)
lock.acquire(timeout=0)
# get local sequence number
def get_sequence_num(s):
for line in s.split("\n"):
(key, sep, value) = line.partition("=")
if key.strip() == "sequenceNumber":
return int(value)
try:
print(os.path.join(orig_diff_path, "state.txt"))
f = open(os.path.join(orig_diff_path, "state.txt"), "r")
begin_sequence = get_sequence_num(f.read())
f.close()
except IOError:
lock.release()
raise
# get remote sequence number
try:
f = urllib.request.urlopen(os.path.join(remote_diff_url, "state.txt"))
server_state = f.read().decode("utf-8")
except IOError:
lock.release()
raise
end_sequence = min(begin_sequence + 10000, get_sequence_num(server_state))
if wanted_end_sequence:
end_sequence = min(end_sequence, wanted_end_sequence)
f.close()
try:
begin_sequence = int(begin_sequence)
end_sequence = int(end_sequence)
except TypeError:
lock.release()
raise
# download diffs, and apply the polygon on them
for i in range(begin_sequence + 1, end_sequence + 1):
print(time.strftime("%Y-%m-%d %H:%M:%S"), i)
for path in [orig_diff_path] + modif_diff_path + [bbox_diff_path]:
tmp_path = os.path.join(
path, "%03d/%03d" % (i // (1000 * 1000), (i // 1000) % 1000))
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
file_location = "%03d/%03d/%03d" % (i // (1000 * 1000),
(i // 1000) % 1000, i % 1000)
# download diff file
print(time.strftime("%Y-%m-%d %H:%M:%S"), " download diff")
orig_diff_file = os.path.join(orig_diff_path, file_location)
for ext in (".osc.gz", ".state.txt"):
try:
(filename, headers) = urllib.request.urlretrieve(
os.path.join(remote_diff_url, file_location) + ext,
orig_diff_file + ext)
except IOError:
lock.release()
raise
file_date = time.mktime(
dateutil.parser.parse(headers["Last-Modified"]).astimezone(
dateutil.tz.tzlocal()).timetuple())
os.utime(orig_diff_file + ext, (file_date, file_date))
if not skip_diff_generation:
generate_bbox_diff(orig_diff_path, file_location, file_date,
bbox_diff_path)
if multiproc_enabled:
lock_num_launched.acquire()
for country in top_countries:
country_param = countries_param[country]
num_launched += 1
if multiproc_enabled:
pool_jobs.append(
pool.apply_async(
generate_diff,
(bbox_diff_path, file_location, file_date,
country_param[0], country_param[1], country),
callback=launch_dep_countries))
else:
pool_jobs = generate_diff(bbox_diff_path, file_location,
file_date, country_param[0],
country_param[1], country)
launch_dep_countries(pool_jobs)
if multiproc_enabled:
lock_num_launched.release()
while True:
lock_num_launched.acquire()
local_num_launched = num_launched
lock_num_launched.release()
if local_num_launched == 0 and len(pool_jobs) == 0:
break
for r in pool_jobs:
r.get()
pool_jobs.remove(r)
assert num_launched == 0
# update osmbin
print(time.strftime("%Y-%m-%d %H:%M:%S"), " update osmbin")
diff_read = OsmSax.OscSaxReader(orig_diff_file + ".osc.gz")
o = OsmBin.OsmBin("/data/work/osmbin/data", "w")
diff_read.CopyTo(o)
del o
del diff_read
# update symbolic links to state.txt
print(time.strftime("%Y-%m-%d %H:%M:%S"),
" update links to state.txt")
update_symlink(orig_diff_file + ".state.txt",
os.path.join(orig_diff_path, "state.txt"))
os.utime(os.path.join(orig_diff_path, "state.txt"),
(file_date, file_date))
sys.stdout.flush()
if multiproc_enabled:
pool.close()
pool.join()
# free lock
sys.stdout.flush()
lock.release()
def analyser(self):
## result file
outxml = OsmSax.OsmSaxWriter(open(self.config.dst, "w"), "UTF-8")
outxml.startDocument()
outxml.startElement(
"analyser",
{"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())})
outxml.startElement("class", {"id": "1", "item": "1040"})
outxml.Element(
"classtext", {
"lang": "fr",
"title": "Polygone non valide (analyse gis)",
"menu": "polygone non valide"
})
outxml.Element(
"classtext", {
"lang": "en",
"title": "Invalid polygon (gis analysis)",
"menu": "invalid polygon"
})
outxml.endElement("class")
## sql querry
sql = """
SELECT osm_id,
name,
astext(st_transform(selfinter,4020)) AS selfinter,
astext(st_transform(way,4020)) AS way
FROM (
SELECT osm_id,name,
st_difference(
st_endpoint(
st_union(
st_exteriorring(way),
st_endpoint(st_exteriorring(way))
)
),
st_endpoint(st_exteriorring(way))
) AS selfinter,
way
FROM %s_polygon
WHERE st_isvalid(way)='f'
) AS tmp
WHERE st_isempty(selfinter)='f'
;
""" % self.config.db_schema
gisconn = psycopg2.connect(self.config.db_string)
giscurs = gisconn.cursor()
giscurs.execute(sql)
apiconn = OsmGis.OsmGis(self.config.db_string, self.config.db_schema)
## format results to outxml
while True:
many = giscurs.fetchmany(1000)
if not many:
break
for res in many:
outxml.startElement("error", {"class": "1"})
for loc in self.get_points(res[2]):
outxml.Element("location", loc)
#outxml.Element("text", {"lang":"en", "value":get_error_text(res[0])})
if res[0] < 0:
outxml.RelationCreate(apiconn.RelationGet(-res[0]))
else:
outxml.WayCreate(apiconn.WayGet(res[0]))
outxml.endElement("error")
outxml.endElement("analyser")
outxml._out.close()
def save(db, lang):
json = request.json
if 'tag' not in json:
abort(422)
# Changeset tags
tags = json['tag']
if 'comment' not in tags or tags['comment'].strip() == '':
tags['comment'] = u'Fixed with Osmose'
if 'source' not in tags or tags['source'].strip() == '':
tags['source'] = u'Osmose'
if 'type' not in tags or tags['type'].strip() == '':
tags['type'] = u'fix'
tags['created_by'] = u'Osmose Editor'
reuse_changeset = json.get('reuse_changeset', True) != False
# Get an open changeset
changeset = request.session.get('changeset')
if changeset and not reuse_changeset:
try:
_changeset_close(changeset)
except:
pass
changeset = None
del request.session['changeset']
request.session.save()
elif changeset:
try:
_changeset_update(changeset, tags)
except:
changeset = None
request.session['changeset'] = changeset
request.session.save()
if not changeset:
changeset = _changeset_create(tags)
request.session['changeset'] = changeset
request.session.save()
# OsmChange
out = io.StringIO()
o = OsmSax.OsmSaxWriter(out, "UTF-8")
o.startDocument()
o.startElement('osmChange', {"version": "0.6", "generator": "OsmSax"})
methode = {
'node': o.NodeCreate,
'way': o.WayCreate,
'relation': o.RelationCreate
}
for action in ('modify', 'delete'):
if action in json and len(json[action]) > 0:
o.startElement(action, {})
for e in json[action]:
try:
ee = utils.fetch_osm_elem(e['type'], e["id"])
except:
ee = None
if ee and ee['version'] == int(e['version']):
ee[u'changeset'] = changeset
ee['tag'] = e['tags']
methode[e['type']](ee)
else:
# FIXME reject
pass
o.endElement(action)
o.endElement('osmChange')
osmchange = out.getvalue()
# Fire the changeset
_changeset_upload(changeset, osmchange)