def main():
file_path = sys.argv[1]
if not os.path.exists(file_path):
print("{} doesn't exist.".format(file_path))
exit(1)
else:
print("Extracting buildings from |{}|...".format(file_path))
root = et.Element('environment')
comment = et.Comment(
'Generated obstacle configurations for {}'.format(file_path))
root.append(comment)
coord_parser = CoordGeo()
p = OSMParser(concurrency=4,
coords_callback=coord_parser.coords_callback)
p.parse(file_path)
bldg_parser = BuildingShapes(coord_parser.coords)
p = OSMParser(concurrency=4, ways_callback=bldg_parser.ways_callback)
p.parse(file_path)
print('# Coords: {}'.format(len(coord_parser.coords)))
print('# Buildings: {}'.format(len(bldg_parser.xml_elmts)))
# Extent the children to the root element
root.extend(bldg_parser.xml_elmts)
#print(prettify(root))
# Dump to file
print('Writing result to {}'.format(output_file))
output = et.ElementTree(root)
output.write(output_file, pretty_print=True)
def __init__(self,
interface,
osmPbfFile,
callback_type=None,
repository=None):
if not isinstance(interface, IOSMPbfParser):
raise Exception("Bad Interface")
self.osmPbfFile = osmPbfFile
if (callback_type == "nodes_callback"):
self.parser = OSMParser(concurrency=4,
nodes_callback=self.fetchNodes)
print("Initializing Nodes Callback......")
elif (callback_type == "ways_callback"):
self.parser = OSMParser(concurrency=4,
ways_callback=self.fetchWays)
print("Initializing Ways Callback......")
elif (callback_type == "relations_callback"):
self.parser = OSMParser(concurrency=4,
relations_callback=self.fetchRelations)
print("Initializing Relations Callback......")
elif (callback_type == "coords_callback"):
self.parser = OSMParser(concurrency=4,
coords_callback=self.fetchCoords)
print("Initializing Coords Callback......")
else:
self.parser = OSMParser()
self.repository = repository
def read(fname, bb=None, ll_bb=None):
g = graph.Graph()
vertex_map = {}
def coords_callback(clist):
for osmid, lon, lat in clist:
p = geom.FPoint(lon, lat)
if ll_bb is not None and not ll_bb.contains(p):
continue
p = coords.lonLatToMeters(p).to_point()
if bb is not None and not bb.contains(p):
continue
vertex_map[osmid] = g.add_vertex(p)
def ways_callback(ways):
for osmid, tags, refs in ways:
if 'highway' not in tags or tags['highway'] in BLACKLIST:
continue
for i in range(len(refs) - 1):
src = refs[i]
dst = refs[i + 1]
if src not in vertex_map or dst not in vertex_map:
continue
g.add_bidirectional_edge(vertex_map[src], vertex_map[dst])
p = OSMParser(coords_callback=coords_callback)
p.parse(fname)
p = OSMParser(ways_callback=ways_callback)
p.parse(fname)
return g
def retrieve_highway(osm_file, selected_zone, tolerance, Ncore=1):
# Parses the OSM file.
point = Point(selected_zone, tolerance)
p = OSMParser(concurrency=Ncore, coords_callback=point.select)
p.parse(osm_file)
highway = Highway(point)
p = OSMParser(concurrency=Ncore, ways_callback=highway.select)
p.parse(osm_file)
highway.point_set = set([item for refs in highway.point for item in refs])
# Getting all coordinates.
point_coordinate = []
for way in highway.point_set:
try:
point_coordinate.append(point.coordinate[way])
except:
pass
# Defining the set of OSM way id and coordinates.
highway_coordinate = []
highway_osmid = []
for refs, osmid in zip(highway.point, highway.osmid):
try:
highway_coordinate.append([point.coordinate[n] for n in refs])
highway_osmid.append(osmid)
except:
pass
return highway_coordinate, highway_osmid
def load_network(self, pbf, filterfunction=lambda r: True):
# read data of public transport network
# required for validating and displaying
self.relation_filter = filterfunction
# get modification time of data source
# TODO: how to determine time when reading from multiple sources?
self.mtime = datetime.datetime.fromtimestamp(
os.stat(pbf)[stat.ST_MTIME])
# first pass:
# collect all interesting relations
print "Collecting relations..."
p = OSMParser(concurrency=4, relations_callback=self.relations_cb)
p.parse(pbf)
# second pass:
# collect ways for these relations
print "Collecting %i ways..." % len(self.ways)
p = OSMParser(concurrency=4, ways_callback=self.ways_cb)
p.parse(pbf)
# collect nodes for collected relations and ways
print "Collecting %i nodes..." % len(self.nodes)
p = OSMParser(concurrency=4, nodes_callback=self.nodes_cb)
p.parse(pbf)
def process_file(filename,
baseurl='http://download.geofabrik.de/osm/europe/',
parentCountry=""):
print "retrieving file " + filename + "..."
urllib.urlretrieve(baseurl + filename, '/tmp/' + filename)
global country
refsSet = set()
nodesLookup = dict()
if parentCountry == "":
country = filename[0:-8].replace('_', ' ').title()
if country in countriesLookup:
country = countriesLookup[country]
print "parsing for refs..."
parser = OSMParser(ways_callback=refs_handler)
parser.parse('/tmp/' + filename)
print "parsing for coords..."
parser = OSMParser(coords_callback=coords_handler)
parser.parse('/tmp/' + filename)
print "parsing for ways..."
parser = OSMParser(ways_callback=ways_handler)
parser.parse('/tmp/' + filename)
scraperwiki.sqlite.save_var(filename, 1)
def __init__(self, path, offset):
self.__processor = DataProcessor()
self.__path = path
way = OSMParser(concurrency=4,
ways_callback=self.__processor.callback_ways)
coord = OSMParser(concurrency=4,
coords_callback=self.__processor.callback_coords)
way.parse(path)
coord.parse(path)
self.__structure = self.__processor.pre_processing(offset)
def load_file(self, filename):
# Reinitialize if we have a new file
ways = []
coords = {}
num_coords = 0
num_ways = 0
# status output
if self.verbose:
sys.stderr.write("\nLoading ways, each '-' is 100 ways, each row is 10,000 ways\n")
p = OSMParser(ways_callback=self.ways_callback)
p.parse(filename)
# status output
if self.verbose:
sys.stderr.write("\n{} ways matched in {} {mem:.1f}MB memory used,\n{} coordinates will be loaded, each '.' is 1% complete\n".format(len(self.ways), filename, len(self.coords), mem=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1048576))
total = len(self.coords)
if total < 100:
self.coords_marker = 1
else:
self.coords_marker = round(total/100)
p = OSMParser(coords_callback=self.coords_callback)
p.parse(filename)
# status output
if self.verbose:
sys.stderr.write("\nCoordinates loaded {mem:.1f}MB memory used.".format(mem=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1048576))
sys.stderr.flush()
# Join routes end-to-end and add them to the way list.
self.join_ways()
# status output
if self.verbose:
sys.stderr.write("\nJoining complete. {mem:.1f}MB memory used.".format(mem=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1048576))
sys.stderr.write("\nCalculating curvature, each '.' is 1% complete\n")
sys.stderr.flush()
# Loop through the ways and calculate their curvature
start_time = time.time()
self.calculate()
# status output
if self.verbose:
sys.stderr.write("\nCalculation complete, {mem:.1f}MB memory used".format(mem=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1048576))
sys.stderr.write('\nCalculation completed in {time:.1f} seconds'.format(time=(time.time() - start_time)))
sys.stderr.flush()
def parse_import(osm_file):
logger.info('Starting to read {}'.format(osm_file))
osm_importer = OsmImporter()
logger.info('Parsing and importing nodes...')
nodes = OSMParser(concurrency=ops_settings['concurrent_workers'],
nodes_callback=osm_importer.parse_nodes)
nodes.parse(osm_file)
logger.info('Parsing relations...')
relations = OSMParser(concurrency=ops_settings['concurrent_workers'],
relations_callback=osm_importer.parse_relations)
relations.parse(osm_file)
logger.info('Found {} ways in relations'.format(
osm_importer.relations_cnt))
logger.info('Parsing ways...')
ways = OSMParser(concurrency=ops_settings['concurrent_workers'],
ways_callback=osm_importer.parse_ways)
ways.parse(osm_file)
logger.info('Found {} ways'.format(osm_importer.ways_cnt))
del osm_importer.relation_ways
# Sort the ways by the first osm_id reference, saves memory for parsing coords
osm_importer.process_ways.sort(key=lambda x: x.refs[0])
# init self.process_ways_length before the first call of parse_coords function!
osm_importer.process_ways_length = len(osm_importer.process_ways)
# https://docs.python.org/3/library/collections.html#collections.deque
osm_importer.process_ways = deque(osm_importer.process_ways)
logger.info('Importing ways... (note this wont work concurrently)')
coords = OSMParser(concurrency=1,
coords_callback=osm_importer.parse_coords_for_ways)
coords.parse(osm_file)
logger.info('Storing remaining pois')
osm_importer.save_remainder()
logger.info('Found {} pois'.format(osm_importer.pois_cnt))
logger.info('Finished import of {}'.format(osm_file))
# logger.debug('Heap: {}'.format(h.heap()))
# clear memory
del osm_importer
def parse(fname, bb=None, ll_bb=None):
from imposm.parser import OSMParser
vertex_set = set([])
vertices = deque([]) # (osm_id, lon, lat)
used_vertices = set([])
out_ways = deque([])
edges = deque([]) # (osm_id, index, src, dst)
def coords_callback(clist):
for osmid, lon, lat in clist:
p = geom.FPoint(lon, lat)
if ll_bb is not None and not ll_bb.contains(p):
continue
p = coords.lonLatToMeters(p).to_point()
if bb is not None and not bb.contains(p):
continue
vertices.append((osmid, np.double(lon), np.double(lat)))
vertex_set.add(osmid)
def ways_callback(ways):
for osmid, tags, refs in ways:
if 'highway' not in tags or tags['highway'] in BLACKLIST:
continue
for i in range(len(refs) - 1):
src = refs[i]
dst = refs[i + 1]
if src not in vertex_set or dst not in vertex_set:
continue
edges.append((osmid, i, src, dst))
used_vertices.add(src)
used_vertices.add(dst)
out_ways.append((osmid, refs))
p = OSMParser(coords_callback=coords_callback)
p.parse(fname)
p = OSMParser(ways_callback=ways_callback)
p.parse(fname)
final_vertices = deque([])
for v in vertices:
(osm_id, _, _) = v
if osm_id in used_vertices:
final_vertices.append(v)
return final_vertices, edges, out_ways
def __init__(self, filename):
self.nodes = {}
self.coords = {}
self.positions = {}
self.ways = {}
self.edges = []
parser = OSMParser(concurrency=1,
ways_callback=self.ways_cb,
nodes_callback=self.nodes_cb,
coords_callback=self.coords_cb)
parser.parse(filename)
for osmid in self.ways:
tags, refs = self.ways[osmid]
if not 'highway' in tags:
continue
# unpack the ways in to edges that will be used for the graphs later
# we store the osm id of the way in the edge, but prolly not needed
for i in xrange(len(refs) - 1):
from_node, to_node = refs[i], refs[i + 1]
from_coords, _ = self.coords[from_node]
to_coords, _ = self.coords[to_node]
d = dist(from_coords, to_coords)
# FIXME: make a class or namedtuple from edge
self.edges.append(
(osmid, from_node, to_node, from_coords, to_coords, d))
def roadsGraph(filename):
roads = Roads()
osmParser = OSMParser(concurrency=4,
coords_callback=roads.nodes,
ways_callback=roads.edges)
osmParser.parse(filename)
return roads.getGraph()
def extractCities(countries_dir, output_dir):
print "extracting cities"
global CC
global citiesFile
if os.path.exists(cities_path):
os.remove(cities_path)
citiesFile = codecs.open(cities_path, encoding='utf-8', mode='wb')
prepare_CC_map()
p = OSMParser(concurrency=4, nodes_callback=handle_nodes)
for dirname, dirnames, filenames in os.walk(countries_dir):
# for subdirname in dirnames:
baseDirName = os.path.basename(dirname)
basePath = os.path.dirname(dirname)
for country_file in filenames:
if country_file != ".DS_Store":
country = os.path.splitext(os.path.basename(country_file))[0]
CC = map_to_CC(country)
# print country + " " + CC
inputFile = dirname + "/" + country_file
print inputFile
p.parse(inputFile)
citiesFile.close()
print "done"
def Parse(xml_file_name, _higherid):
# id of a higher element in the map hierarchy of Guided Area
# (id of the area that contains the current elements)
higherid = _higherid
buildings = WaysOsmIdExtractor()
p = OSMParser(concurrency=4,
ways_callback=buildings.border,
nodes_callback=buildings.points)
p.parse_xml_file(xml_file_name)
b = OSMParser(concurrency=4, coords_callback=coords_callback)
b.parse_xml_file(xml_file_name)
return Data
def buildingsPolygons(filename):
buildings = Buildings()
osmparser = OSMParser(concurrency=4,
coords_callback=buildings.nodes,
ways_callback=buildings.osmways,
relations_callback=buildings.osmrelations)
osmparser.parse(filename)
return buildings.getPolygons()
def __init__(self):
print "Reading OpenStreetMap data from file=", config['osm_data']
# data = Graph(config['osm_data'])
data = Graph()
p = OSMParser(concurrency=4, ways_callback=data.ways)
p.parse(config['osm_data'])
print data.highways
def process_file(filename):
refsSet = set()
nodesLookup = dict()
print "parsing for refs..."
parser = OSMParser(ways_callback=refs_handler,
relations_callback=rels_handler)
parser.parse('/tmp/' + filename)
print "parsing for coords..."
parser = OSMParser(coords_callback=coords_handler)
parser.parse('/tmp/' + filename)
print "parsing for ways and nodes..."
parser = OSMParser(ways_callback=ways_handler,
nodes_callback=nodes_handler)
parser.parse('/tmp/' + filename)
def __init__(self, osmfile):
# parse the input file and save its contents in memory
# initialize street network
self.street_network = StreetNetwork()
# coord pairs as returned from imposm
self.coords = dict()
# max and min latitude and longitude
self.bounds = dict()
self.bounds["min_lat"] = 9999
self.bounds["max_lat"] = -9999
self.bounds["min_lon"] = 9999
self.bounds["max_lon"] = -9999
# active copy of OSM data indexed by OSM id
self.all_osm_relations = dict()
self.all_osm_ways = dict()
self.all_osm_nodes = dict()
# nodes with specific landuse tags
self.residential_nodes = set()
self.industrial_nodes = set()
self.commercial_nodes = set()
# subset that is also connected to the street network
self.connected_residential_nodes = set()
self.connected_industrial_nodes = set()
self.connected_commercial_nodes = set()
# mapping from highway types to max speeds
# we do this so there"s always a speed limit for every edge, even if
# none is in the OSM data
self.max_speed_map = dict()
self.max_speed_map["motorway"] = 140
self.max_speed_map["trunk"] = 120
self.max_speed_map["primary"] = 100
self.max_speed_map["secondary"] = 80
self.max_speed_map["tertiary"] = 70
self.max_speed_map["road"] = 50
self.max_speed_map["minor"] = 50
self.max_speed_map["unclassified"] = 50
self.max_speed_map["residential"] = 30
self.max_speed_map["track"] = 30
self.max_speed_map["service"] = 20
self.max_speed_map["path"] = 10
self.max_speed_map["cycleway"] = 1 # >0 to prevent infinite weights
self.max_speed_map["bridleway"] = 1 # >0 to prevent infinite weights
self.max_speed_map["pedestrian"] = 1 # >0 to prevent infinite weights
self.max_speed_map["footway"] = 1 # >0 to prevent infinite weights
p = OSMParser(concurrency = 1,
coords_callback = self.coords_callback,
nodes_callback = self.nodes_callback,
ways_callback = self.ways_callback,
relations_callback = self.relations_callback)
p.parse(osmfile)
def main():
file_name = sys.argv[1]
filter = AmenityFilter()
nodes = NodesCoords()
p = OSMParser(nodes_callback=nodes.nodes_callback,
nodes_tag_filter=filter.amenity_filter)
p.parse_pbf_file(file_name)
print yaml.dump(nodes.points)
def station_nodes_from_nodes():
def imposm_extract_railway_station_nodes_from_nodes(nodes):
target_node_tags=["station"]
excluding_node_tags=["light_rail","subway"]
for osmid, tags, (lon,lat) in nodes:
if tags.get("railway","") in target_node_tags and tags.get("station","") not in excluding_node_tags:
station_dict[osmid]=[tags.get("name","N/A"),lat,lon]
station_dict_complete[osmid]=[tags, lat, lon]
OSMParser(concurrency=CONCURRENCYVAL, nodes_callback=imposm_extract_railway_station_nodes_from_nodes).parse(FOLDER+FILE)
def load_from_file(fname):
ways = Ways()
p = OSMParser(concurrency=4, ways_callback=ways.get_ways)
p.parse(fname)
coords = Coords()
p = OSMParser(concurrency=4, coords_callback=coords.get_coords)
p.parse(fname)
id2coords = coords.coords
roads = []
for tags, refs, weight in ways.ways.values():
for i in range(1, len(refs)):
road = (id2coords[refs[i - 1]][0], id2coords[refs[i - 1]][1],
id2coords[refs[i]][0], id2coords[refs[i]][1], weight)
roads.append(road)
print len(roads)
return RoadNetwork2(roads)
def station_nodes_from_ways():
def imposm_extract_railway_station_nodes_from_ways(ways):
'''get accurate railway stations from exact railways ways instead of getting wrong subway stations'''
target_way_tags=["rail"]
for osmid, tags, refs in ways:
if tags.get("railway","") in target_way_tags:
for i in refs:
if i in station_dict:
station_dict[i].append("railwaysuffix")
OSMParser(concurrency=CONCURRENCYVAL, ways_callback=imposm_extract_railway_station_nodes_from_ways).parse(FOLDER+FILE)
def station_nodes_from_relations():
def imposm_extract_railway_stations_from_relations(relations):
'''get accurate railway stations from exact railways relations instead of getting wrong subway stations'''
target_relation_tags=["train","rail"]
for osmid, tags, members in relations:
if tags.get("route") in target_relation_tags or tags.get("railway") in target_relation_tags:
for i in members:
if i[1]=="node" and (i[0] in station_dict):
station_dict[i[0]].append("railwaysuffix")
OSMParser(concurrency=CONCURRENCYVAL, relations_callback=imposm_extract_railway_stations_from_relations).parse(FOLDER+FILE)
def __init__(self, osmfile):
self.nodes = dict()
self.bounds = dict()
self.bounds["min_lat"] =-9999 # 58.3773000
self.bounds["max_lat"] = 9999 # 58.4236000
self.bounds["min_lng"] =-9999 # 15.5207000
self.bounds["max_lng"] = 9999 # 15.6248000
p=OSMParser(coords_callback = self.coords_callback)
p.parse(osmfile)
def reload_position_data(self):
print "Reloading (OSM) position data"
osm_stations = BusStationGetter()
p = OSMParser(concurrency=4, nodes_callback=osm_stations.nodes)
p.parse(OSM_file)
osm_stations = sorted(osm_stations.stations, key=itemgetter(0))
lpp_stations = sorted(get_lpp_stations(), key=itemgetter(0))
stations = get_coordinates_for_stations(lpp_stations, osm_stations)
pickle.dump(stations, open("lpp-station-geo-data.pypickle", "wb"))
print "Reloaded position data"
def import_file(self, file_name):
"""Parses a file and feeds and imports the parsed osm primitives
into a remote crdb cluster
Parameters:
file_name -- the name of the input file
"""
parser = OSMParser(concurrency=10,
nodes_callback=self._on_nodes,
ways_callback=self._on_ways,
relations_callback=self._on_relations)
parser.parse(file_name)
def test_save_bank(capfd):
"""Test saving a bank node
"""
# set up to parse `bank.osm`
input_filepath = os.path.join(os.path.dirname(__file__),"data","bank.osm")
node_handler = NodeHandler()
p = OSMParser(nodes_callback=node_handler.nodes)
p.parse(input_filepath)
# don't use db, so expect tab-separated STDOUT
resout, reserr = capfd.readouterr()
assert resout == '1\tBank of Testing\tbank\tPOINT(10.0 50.0)\n'
def collect_subway_nodes():
def imposm_extract_subway_nodes_from_relations(relations):
target_relation_tags=["subway","light_rail","tram"]
for osmid, tags, members in relations:
if tags.get("route","") in target_relation_tags or tags.get("railway") in target_relation_tags:
for m in members:
if m[1]=="node":
subway_nodes_complete.add(m[0])
if m[1]=="way":
subway_ways_from_relations.add(m[0])
def imposm_extract_subway_nodes_from_ways(ways):
target_way_tags=["subway","light_rail","tram"]
for osmid, tags, refs in ways:
if tags.get("railway","") in target_way_tags or osmid in subway_ways_from_relations:
for r in refs:
subway_nodes_complete.add(r)
OSMParser(concurrency=CONCURRENCYVAL, relations_callback=imposm_extract_subway_nodes_from_relations).parse(FOLDER+FILE)
OSMParser(concurrency=CONCURRENCYVAL, ways_callback=imposm_extract_subway_nodes_from_ways).parse(FOLDER+FILE)
def __call__(self, filename):
client.init(splitext(basename(filename))[0], self.level)
if self.drop:
client.dropdb(self.level)
parser = OSMParser(concurrency=4,
ways_callback=self.way,
coords_callback=self.coord,
nodes_callback=self.node,
relations_callback=self.relation)
parser.parse(filename)
self.flush_buffer()
def station_nodes_from_areas():
def imposm_extract_railway_station_nodes_from_areas(areas):
target_area_tags=["station"]
excluding_area_tags=["light_rail","subway"]
for osmid, tags, refs in areas:
'''areas are using the same key as ways in OpenStreetMap, while areas are "closed ways"'''
if tags.get("railway","") in target_area_tags and tags.get("station","") not in excluding_area_tags:
station_dict[refs[0]]=[tags.get("name","N/A")]
station_dict_complete[refs[0]]=[tags]
'''the substitution node for the area of the station is to be refined'''
def imposm_nodes_substitute_for_area_stations(coords):
'''must running after imposm_extract_station_from_areas for some induced station substitution nodes are not extracted by OSMParser callbacks'''
for osmid, lon, lat in coords:
if osmid in station_dict and len(station_dict[osmid])==1:
station_dict[osmid].append(lat)
station_dict[osmid].append(lon)
station_dict[osmid].append("railwaysuffix")
station_dict_complete[osmid].append(lat)
station_dict_complete[osmid].append(lon)
station_dict_complete[osmid].append("railwaysuffix")
OSMParser(concurrency=CONCURRENCYVAL, ways_callback=imposm_extract_railway_station_nodes_from_areas).parse(FOLDER+FILE)
OSMParser(concurrency=CONCURRENCYVAL, coords_callback=imposm_nodes_substitute_for_area_stations).parse(FOLDER+FILE)
