def test_serialize(self):
data = osm.Osm()
n = data.Node(4, 0)
n.tags['entrance'] = 'yes'
n.tags['addr:street'] = "Calle la Ñ"
n.tags['addr:housenumber'] = '7'
w = data.Way([(12, 0), (14, 0), (14, 2), (12, 2), (12, 0)])
w.tags['leisure'] = 'swiming_pool'
r = data.MultiPolygon([[[(0, 0), (10, 0), (10, 6), (0, 6), (0, 0)],
[(8, 1), (9, 1), (9, 2), (8, 2), (8, 1)]]])
r.tags['building'] = 'residential'
fo = StringIO()
osmxml.serialize(fo, data)
result = bytes(fo.getvalue().encode('utf-8'))
root = etree.fromstring(result)
for (xmlnode, osmnode) in zip(root.findall('node'), data.nodes):
self.assertEqual(float(xmlnode.get('lon')), osmnode.x)
self.assertEqual(float(xmlnode.get('lat')), osmnode.y)
for (xmltag, osmtag) in zip(root.findall('node/tag'), n.tags.items()):
self.assertEqual(xmltag.get('k'), osmtag[0])
self.assertEqual(xmltag.get('v'), osmtag[1])
nw = 0
for (xmlway, osmway) in zip(root.findall('way'), data.ways):
nw += 1
for (xmlnd, osmnd) in zip(xmlway.findall('nd'), osmway.nodes):
self.assertEqual(int(xmlnd.get('ref')), osmnd.id)
self.assertEqual(nw, 3)
for (xmltag, osmtag) in zip(root.findall('way/tag'), w.tags.items()):
self.assertEqual(xmltag.get('k'), osmtag[0])
self.assertEqual(xmltag.get('v'), osmtag[1])
for (i, (xmlm, osmm)) in enumerate(
zip(root.findall('relation/member'), r.members)):
self.assertEqual(int(xmlm.get('ref')), osmm.ref)
self.assertEqual(xmlm.get('role'), 'outer' if i == 0 else 'inner')
for (xmltag, osmtag) in zip(root.findall('relation/tag'),
r.tags.items()):
self.assertEqual(xmltag.get('k'), osmtag[0])
self.assertEqual(xmltag.get('v'), osmtag[1])
self.assertEqual(sum([1 for r in root.findall('relation')]), 1)
self.assertEqual(root.find('note'), None)
self.assertEqual(root.find('meta'), None)
data.note = 'foobar'
data.meta = {'foo': 'bar'}
data.tags['type'] = 'import'
fo = StringIO()
osmxml.serialize(fo, data)
result = bytes(fo.getvalue().encode('utf-8'))
root = etree.fromstring(result)
self.assertEqual(root.find('note').text, 'foobar')
self.assertEqual(root.find('meta').get('foo'), 'bar')
for (xmltag, osmtag) in zip(root.findall('changeset/tag'),
data.tags.items()):
self.assertEqual(xmltag.get('k'), osmtag[0])
self.assertEqual(xmltag.get('v'), osmtag[1])
def test_replace(self):
n1 = self.d.Node(1, 1)
d2 = osm.Osm()
n2 = d2.Node(2, 2)
p = self.d.parents[n1]
self.d.replace(n1, n2)
self.assertNotIn(n1, self.d.elements)
self.assertIn(n2, self.d.elements)
self.assertEqual(n2.container, self.d)
self.assertEqual(self.d.get(n2.id), n2)
self.assertEqual(self.d.parents[n2], p)
def test_address_stats(self):
ad = osm.Osm()
ad.Node(0,0, {'addr:street': 's1'})
ad.Node(2,0, {'addr:street': 's2', 'entrance': 'yes'})
ad.Node(4,0, {'addr:place': 'p1', 'entrance': 'yes'})
ad.Way([], {'addr:street': 's3'})
r = report.Report()
r.address_stats(ad)
self.assertEqual(r.out_addr_str, 3)
self.assertEqual(r.out_addr_plc, 1)
self.assertEqual(r.out_address_entrance, 2)
self.assertEqual(r.out_address_building, 2)
def test_get_current_ad_osm(self, m_report, m_log):
d = osm.Osm()
d.Node(0, 0, {'addr:housenumber': '12', 'addr:street': 'foobar'})
d.Node(1, 1, {'addr:housenumber': '14', 'addr:street': 'foobar'})
d.Node(2, 2, {'addr:housenumber': '10', 'addr:place': 'bartaz'})
self.m_app.get_current_ad_osm = cat.CatAtom2Osm.get_current_ad_osm.__func__
self.m_app.read_osm.return_value = d
address = self.m_app.get_current_ad_osm(self.m_app)
self.assertEquals(address, set(['foobar14', 'foobar12', 'bartaz10']))
self.assertNotIn('osm_addresses_whithout_number', m_report)
d.Node(3, 3, {'addr:street': 'x'})
d.Node(4, 4, {'addr:place': 'y'})
self.m_app.read_osm.return_value = d
address = self.m_app.get_current_ad_osm(self.m_app)
self.assertEquals(m_report.osm_addresses_whithout_number, 2)
def test_cons_stats(self):
r = report.Report()
r.building_counter = Counter()
data = osm.Osm()
data.Node(0,0, {'leisure': 'swimming_pool'})
data.Node(0,0, {'building': 'a', 'fixme': 'f1'})
data.Node(0,0, {'building': 'b', 'fixme': 'f2'})
data.Node(0,0, {'building:part': 'yes', 'fixme': 'f2'})
data.Node(0,0)
r.cons_stats(data)
self.assertEqual(r.out_pools, 1)
self.assertEqual(r.out_buildings, 2)
self.assertEqual(r.out_parts, 1)
self.assertEqual(r.building_counter['a'], 1)
self.assertEqual(r.building_counter['b'], 1)
self.assertEqual(r.fixme_counter['f1'], 1)
self.assertEqual(r.fixme_counter['f2'], 2)
def test_write_osm(self, m_gz, m_codecs, m_xml, m_os):
m_os.path.join = lambda *args: '/'.join(args)
m_xml.serialize.return_value = 'taz'
data = osm.Osm()
data.Node(0, 0, {'ref': '1'})
data.Node(1, 1, {'ref': '2'})
data.Node(2, 2)
self.m_app.write_osm = cat.CatAtom2Osm.write_osm.__func__
self.m_app.write_osm(self.m_app, data, 'bar')
self.assertNotIn('ref',
[k for el in data.elements for k in el.tags.keys()])
m_codecs.open.assert_called_once_with('foo/bar', 'w', 'utf-8')
file_obj = m_codecs.open.return_value
m_xml.serialize.assert_called_once_with(file_obj, data)
m_xml.reset_mock()
self.m_app.write_osm(self.m_app, data, 'bar', compress=True)
m_gz.open.assert_called_once_with('foo/bar.gz', 'w')
f_gz = m_gz.open.return_value
m_codecs.EncodedFile.assert_called_once_with(f_gz, 'utf-8')
def read_shp_as_osm(shp_path, add_ele_tag=True):
osm_data = osm.Osm({})
node_by_coord = {}
dst_crs = osgeo.osr.SpatialReference()
dst_crs.ImportFromEPSG(4326)
with fiona.open(shp_path) as src:
src_crs = osgeo.osr.SpatialReference()
src_crs.ImportFromProj4(fiona.crs.to_string(src.crs))
transformation = osgeo.osr.CoordinateTransformation(src_crs, dst_crs)
i = 0
size = len(src)
percent = -1
for item in src:
i = i + 1
new_percent = int(100*(i+1)/size)
if VERBOSE and new_percent > percent:
percent = new_percent
sys.stderr.write("{0} % ({1} / {2})\r".format(percent, i, size))
if item['geometry']['type'] == "LineString":
way = osm_data.create_way(
attrs={},
tags={str(key): str(value)
for key,value in item['properties'].items()
if value is not None})
coordinates = item["geometry"]["coordinates"]
for (x,y,z) in coordinates:
x,y,z = transformation.TransformPoint(x,y,z)
node = node_by_coord.get((x,y))
if node is None:
node = osm_data.create_node(
attrs={'lon':str(x), 'lat':str(y)}, tags={})
node_by_coord[(x,y)] = node
way.add_node(node)
if add_ele_tag:
for node_index in (0, -1):
if coordinates[node_index][2] >= 0 and coordinates[node_index][2] < 5000:
osm_data.nodes[way.nodes[node_index]].tags["ele"] = str(z)
if VERBOSE:
sys.stderr.write("\n")
return osm_data
def setUp(self):
self.d = osm.Osm()
def deserialize(infile, data=None):
"""Generates or append to an OSM data set from OSM XML"""
if data is None:
data = osm.Osm()
context = etree.iterparse(infile, events=('end',))
childs = []
tags = {}
for event, elem in context:
if elem.tag == 'osm':
data.upload = elem.get('upload')
data.version = elem.get('version')
data.generator = elem.get('generator')
elif elem.tag == 'changeset':
data.tags = tags
tags = {}
elif elem.tag == 'note':
data.note = str(elem.text)
elif elem.tag == 'meta':
data.meta = dict(elem.attrib)
elif elem.tag == 'node':
n = data.Node(float(elem.get('lon')), float(elem.get('lat')),
attrs=dict(elem.attrib), tags=tags)
tags = {}
elif elem.tag == 'way':
w = data.Way(attrs=dict(elem.attrib), tags=tags)
w.nodes = childs
childs = []
tags = {}
elif elem.tag == 'nd':
childs.append(elem.get('ref'))
elif elem.tag == 'relation':
r = data.Relation(attrs=dict(elem.attrib), tags=tags)
r.members = childs
childs = []
tags = {}
elif elem.tag == 'member':
childs.append({
'ref': elem.get('ref'),
'type': elem.get('type'),
'role': elem.get('role')
})
elif elem.tag == 'tag':
tags[elem.get('k')] = elem.get('v')
elem.clear()
if hasattr(elem, 'xpath'):
for ancestor in elem.xpath('ancestor-or-self::*'):
while ancestor.getprevious() is not None:
del ancestor.getparent()[0]
del context
for way in data.ways:
missing = []
for i, ref in enumerate(way.nodes):
if isinstance(ref, basestring):
if 'n{}'.format(ref) in data.index:
n = data.get(ref)
way.nodes[i] = n
data.parents[n].add(way)
else:
missing.append(i)
if len(missing) > 0:
for i in sorted(missing, reverse=True):
way.nodes.pop(i)
if way.version is not None:
way.version = str(int(way.version) + 1)
for rel in data.relations:
missing = []
for i, m in enumerate(rel.members):
if isinstance(m, dict):
if m['type'][0].lower() + str(m['ref']) in data.index:
el = data.get(m['ref'], m['type'])
rel.members[i] = osm.Relation.Member(el, m['role'])
data.parents[el].add(rel)
else:
missing.append(i)
if len(missing) > 0:
for i in sorted(missing, reverse=True):
rel.members.pop(i)
if rel.version is not None:
rel.version = str(int(rel.version) + 1)
return data
def run(self):
"""Launches the app"""
log.info(_("Start processing '%s'"), report.mun_code)
self.get_zoning()
if self.options.address:
self.read_address()
if self.is_new:
self.options.tasks = False
self.options.building = False
self.options.zoning = False
self.options.parcel = False
if self.options.zoning or self.options.tasks:
self.process_zoning()
if not self.options.tasks:
self.delete_shp('rustic_zoning.shp')
self.address_osm = osm.Osm()
self.building_osm = osm.Osm()
if self.options.address and not self.is_new and not self.options.manual:
current_address = self.get_current_ad_osm()
self.address.conflate(current_address)
if self.options.building or self.options.tasks:
self.get_building()
self.process_building()
if self.options.address:
self.building.move_address(self.address)
self.building.reproject()
if self.options.tasks:
self.building.set_tasks(self.urban_zoning, self.rustic_zoning)
if not self.options.manual:
current_bu_osm = self.get_current_bu_osm()
if self.building.conflate(current_bu_osm):
fn = os.path.join(self.path, 'current_building')
if not os.path.exists(fn + '.bak.osm'):
shutil.copyfile(fn + '.osm', fn + '.bak.osm')
self.write_osm(current_bu_osm, 'current_building.osm')
del current_bu_osm
report.building_counter = Counter()
if self.options.address:
self.address.reproject()
self.address_osm = self.address.to_osm()
del self.address
self.delete_shp('address.shp')
if self.options.tasks:
self.process_tasks(self.building)
if self.options.zoning:
self.export_layer(self.urban_zoning, 'urban_zoning.geojson',
'GeoJSON')
self.export_layer(self.rustic_zoning, 'rustic_zoning.geojson',
'GeoJSON')
self.rustic_zoning.difference(self.urban_zoning)
self.rustic_zoning.append(self.urban_zoning)
self.export_layer(self.rustic_zoning, 'zoning.geojson', 'GeoJSON')
if hasattr(self, 'urban_zoning'):
del self.urban_zoning
self.delete_shp('urban_zoning.shp')
if hasattr(self, 'rustic_zoning'):
del self.rustic_zoning
self.delete_shp('rustic_zoning.shp')
if self.options.building:
self.building_osm = self.building.to_osm()
del self.building
self.delete_shp('building.shp')
if self.options.address:
self.merge_address(self.building_osm, self.address_osm)
if not self.options.tasks:
report.address_stats(self.building_osm)
if not self.options.tasks:
report.cons_stats(self.building_osm)
self.write_osm(self.building_osm, 'building.osm')
if not self.options.tasks:
report.osm_stats(self.building_osm)
del self.building_osm
elif self.options.tasks:
del self.building
self.delete_shp('building.shp')
if self.options.address:
if not self.options.building and not self.options.tasks:
report.address_stats(self.address_osm)
self.write_osm(self.address_osm, 'address.osm')
del self.address_osm
if self.options.parcel:
self.process_parcel()
self.end_messages()
