def __init__(self, osmfile): # parse the input file and save its contents in memory # road ids and node refs as returned from imposm self.roads = dict() p = OSMParser(ways_callback = self.ways_callback) p.parse(osmfile)
def extractCities( countries_dir, output_dir ): print "extracting cities" global CC global citiesFile if os.path.exists( cities_path ): os.remove( cities_path ) citiesFile = codecs.open( cities_path, encoding='utf-8', mode='wb' ) prepare_CC_map( ) p = OSMParser( concurrency=4, nodes_callback = handle_nodes ) for dirname, dirnames, filenames in os.walk( countries_dir ): # for subdirname in dirnames: baseDirName = os.path.basename( dirname ) basePath = os.path.dirname( dirname ) for country_file in filenames: if country_file != ".DS_Store": country = os.path.splitext( os.path.basename( country_file ) )[0] CC = map_to_CC( country ) # print country + " " + CC inputFile = dirname + "/" + country_file print inputFile p.parse( inputFile ) citiesFile.close() print "done"
def remove_tags(input_filename, output_fp, tags_to_keep, close_output_fp=True):
output_writer = OSMWriter(fp=output_fp)
remover = TagRemover(output_writer, tags_to_keep)
parser = OSMParser(ways_callback=remover.ways, nodes_callback=remover.nodes, concurrency=1)
parser.parse(input_filename)
output_writer.close(close_file=close_output_fp)
def __init__(self, osmfile):
# parse the input file and save its contents in memory
# initialize street network
self.street_network = StreetNetwork()
# coord pairs as returned from imposm
self.coords = dict()
# max and min latitude and longitude
self.bounds = dict()
self.bounds["min_lat"] = 9999
self.bounds["max_lat"] = -9999
self.bounds["min_lon"] = 9999
self.bounds["max_lon"] = -9999
# active copy of OSM data indexed by OSM id
self.all_osm_relations = dict()
self.all_osm_ways = dict()
self.all_osm_nodes = dict()
# nodes with specific landuse tags
self.residential_nodes = set()
self.industrial_nodes = set()
self.commercial_nodes = set()
# subset that is also connected to the street network
self.connected_residential_nodes = set()
self.connected_industrial_nodes = set()
self.connected_commercial_nodes = set()
# mapping from highway types to max speeds
# we do this so there"s always a speed limit for every edge, even if
# none is in the OSM data
self.max_speed_map = dict()
self.max_speed_map["motorway"] = 140
self.max_speed_map["trunk"] = 120
self.max_speed_map["primary"] = 100
self.max_speed_map["secondary"] = 80
self.max_speed_map["tertiary"] = 70
self.max_speed_map["road"] = 50
self.max_speed_map["minor"] = 50
self.max_speed_map["unclassified"] = 50
self.max_speed_map["residential"] = 30
self.max_speed_map["track"] = 30
self.max_speed_map["service"] = 20
self.max_speed_map["path"] = 10
self.max_speed_map["cycleway"] = 1 # >0 to prevent infinite weights
self.max_speed_map["bridleway"] = 1 # >0 to prevent infinite weights
self.max_speed_map["pedestrian"] = 1 # >0 to prevent infinite weights
self.max_speed_map["footway"] = 1 # >0 to prevent infinite weights
p = OSMParser(concurrency = 1,
coords_callback = self.coords_callback,
nodes_callback = self.nodes_callback,
ways_callback = self.ways_callback,
relations_callback = self.relations_callback)
p.parse(osmfile)
def __init__(self, osmfile):
# parse the input file and save its contents in memory
# initialize street network
self.street_network = StreetNetwork()
# coord pairs as returned from imposm
self.coords = dict()
# max and min latitude and longitude
self.bounds = dict()
self.bounds["min_lat"] = 9999
self.bounds["max_lat"] = -9999
self.bounds["min_lon"] = 9999
self.bounds["max_lon"] = -9999
# active copy of OSM data indexed by OSM id
self.all_osm_relations = dict()
self.all_osm_ways = dict()
self.all_osm_nodes = dict()
# nodes with specific landuse tags
self.residential_nodes = set()
self.industrial_nodes = set()
self.commercial_nodes = set()
# subset that is also connected to the street network
self.connected_residential_nodes = set()
self.connected_industrial_nodes = set()
self.connected_commercial_nodes = set()
# mapping from highway types to max speeds
# we do this so there"s always a speed limit for every edge, even if
# none is in the OSM data
self.max_speed_map = dict()
self.max_speed_map["motorway"] = 140
self.max_speed_map["trunk"] = 120
self.max_speed_map["primary"] = 100
self.max_speed_map["secondary"] = 80
self.max_speed_map["tertiary"] = 70
self.max_speed_map["road"] = 50
self.max_speed_map["minor"] = 50
self.max_speed_map["unclassified"] = 50
self.max_speed_map["residential"] = 30
self.max_speed_map["track"] = 30
self.max_speed_map["service"] = 20
self.max_speed_map["path"] = 10
self.max_speed_map["cycleway"] = 1 # >0 to prevent infinite weights
self.max_speed_map["bridleway"] = 1 # >0 to prevent infinite weights
self.max_speed_map["pedestrian"] = 1 # >0 to prevent infinite weights
self.max_speed_map["footway"] = 1 # >0 to prevent infinite weights
p = OSMParser(concurrency=4,
coords_callback=self.coords_callback,
nodes_callback=self.nodes_callback,
ways_callback=self.ways_callback,
relations_callback=self.relations_callback)
p.parse(osmfile)
class p:
def __init__(self):
server = Server()
self.db=server.get_or_create_db("b")
self.i=0
self.parser=OSMParser(nodes_callback=self.coord, ways_callback=self.way, relations_callback=self.rel, coords_callback=self.coord)
def parse(self,x):
self.parser.parse(x)
def coord(self,c):
out=dict()
for osm_id,lat,lng in c:
out["type"]="coord"
out["_id"]=str(osm_id)
out["geometry"]={"type":"Point","coordinates":lng}
self.db["c"+str(osm_id)]=out
self.i=self.i+1
print "cached a coord totoal now "+str(self.i)
def node(self,n):
out=dict()
for osm_id,tags,coord in n:
c=[coord[0],coord[1]]
out["type"]="node"
out["_id"]=str(osm_id)
out["geometry"]={"type":"Point","coordinates":c}
for k in tags:
if k !="type" and k!="_id"and k!="geometry":
out[k]=tags[k]
self.db[str(osm_id)]=out
self.i=self.i+1
print "cached a node totoal now "+str(self.i)
def way(self,w):
out=dict()
for osm_id,tags,mem in w:
out["type"]="way"
out["_id"]=str(osm_id)
out["members"]=mem
for k in tags:
if k !="type" and k!="_id"and k!="members":
out[k]=tags[k]
self.db[str(osm_id)]=out
self.i=self.i+1
print "cached a way totoal now "+str(self.i)
def rel(self,r):
out=dict()
for osm_id,tags,mem in r:
out["type"]="relation"
out["_id"]=str(osm_id)
out["members"]=mem
for k in tags:
if k !="type" and k!="_id"and k!="members":
out[k]=tags[k]
self.db[str(osm_id)]=out
self.i=self.i+1
print "cached a rel totoal now "+str(self.i)
def clean(self):
for k in self.nodes:
if self.coords.has_key(k):
del self.coords[k]
def main():
file_name = sys.argv[1]
filter = AmenityFilter()
nodes = NodesCoords()
p = OSMParser(nodes_callback=nodes.nodes_callback,
nodes_tag_filter=filter.amenity_filter)
p.parse_pbf_file(file_name)
print yaml.dump(nodes.points)
def main():
file_name = sys.argv[1]
filter = AmenityFilter()
nodes = NodesCoords()
p = OSMParser(nodes_callback=nodes.nodes_callback,
nodes_tag_filter=filter.amenity_filter)
p.parse_pbf_file(file_name)
print yaml.dump(nodes.points)
def fetch_and_parse_pbf():
print 'Fetching', PBF_URL
pbf_file = 'brandenburg-latest.osm.pbf'
urllib.urlretrieve(PBF_URL, pbf_file)
np = NodesParser()
p = OSMParser(concurrency=2, nodes_callback=np.nodes)
p.parse(pbf_file)
np.close()
print 'Dataset', DATASET_PATH, 'with', np.count, 'nodes parsed from OSM xml'
def parseOSMfile(filename,waysConf,areasConf,bridgesConf,tunnelsConf,srtm): """ Parse OSM file """ clas = classifier(waysConf,areasConf,bridgesConf,tunnelsConf,srtm) p = OSMParser(concurrency=4, ways_callback=clas.ways_cb, nodes_callback=clas.nodes_cb, relations_callback=clas.relations_cb, coords_callback=clas.coords_cb) p.parse(filename) clas.Map.nodes.sort(key=lambda node: node.id) clas.Map.ways.sort(key=lambda way: way.id) for i in range(len(clas.Map.nodes)): clas.Map.nodesIdx[clas.Map.nodes[i].id]=i return clas.Map
def __init__(self, osm_file, coords, nodes, config):
self.tmp_dir = tempfile.mkdtemp()
self.config = config
self.osm_file = osm_file
self.coords = dict(coords.items() + nodes.items())
osm_parser = OSMParser(concurrency=1, ways_callback=self._ways_callback)
osm_parser.parse(osm_file)
def __init__(self, osmfile):
self.nodes = dict()
self.bounds = dict()
self.bounds["min_lat"] =-9999 # 58.3773000
self.bounds["max_lat"] = 9999 # 58.4236000
self.bounds["min_lng"] =-9999 # 15.5207000
self.bounds["max_lng"] = 9999 # 15.6248000
p=OSMParser(coords_callback = self.coords_callback)
p.parse(osmfile)
def add_churches(filename):
# get churches
churches = GetChurches()
p = OSMParser(concurrency=4,
nodes_callback=churches.nodes,
ways_callback=churches.ways)
p.parse(filename)
update_refs.apply_async(args=[filename])
return True
def calc_all_ways(self):
p = OSMParser(concurrency=4, coords_callback=self._read_coords,
ways_callback=self._read_ways)
if not self.silent:
print("Reading file")
p.parse(self.filepath)
if not self.silent:
print("Summing")
self._calc_ways()
return sum(self.way_distances)
def extract(self,osm_filename):
self.p = OSMParser(ways_callback=self.find_highway_nodes)
self.p.parse(osm_filename)
print "Highway nodes found"
self.p = OSMParser(coords_callback=self.count_nodes)
self.p.parse(osm_filename)
print "Created graph nodes"
self.p = OSMParser(ways_callback=self.connect_nodes)
self.p.parse(osm_filename)
print len(self.edge_endpts)
def load_network(self, pbf, filterfunction=lambda r: True): # read data of public transport network # required for validating and displaying self.relation_filter = filterfunction # get modification time of data source # TODO: how to determine time when reading from multiple sources? self.mtime = datetime.datetime.fromtimestamp(os.stat(pbf)[stat.ST_MTIME]) # first pass: # collect all interesting relations print "Collecting relations..." p = OSMParser(concurrency=4, relations_callback=self.relations_cb) p.parse(pbf) # second pass: # collect ways for these relations print "Collecting %i ways..." % len(self.ways) p = OSMParser(concurrency=4, ways_callback=self.ways_cb) p.parse(pbf) # collect nodes for collected relations and ways print "Collecting %i nodes..." % len(self.nodes) p = OSMParser(concurrency=4, nodes_callback=self.nodes_cb) p.parse(pbf)
def load_osm_features(self, pbf_path , families):
logger.info(" Adding feature from OSM.")
pbf_path, pbf_name = os.path.split(pbf_path)
categories = {}
for family in families:
locations = {}
class FamilyParser(object):
ways_objects = {}
nodes_objects = {}
all_nodes = {}
def ways(self, ways):
for osmid, tags, refs in ways:
for key in families[family].keys():
if key in tags.keys():
if families[family][key] == "*" or tags[key] in families[family][key]:
self.ways_objects[int(osmid)] = {"nd" : refs, "tag" : tags[key]}
def nodes(self, nodes):
for osmid, tags, coords in nodes:
for key in families[family].keys():
if key in tags.keys():
if families[family][key] == "*" or tags[key] in families[family][key]:
self.nodes_objects[int(osmid)] = {"lon" : coords[0], "lat" : coords[1], "tag" : tags[key]}
def coords(self, coords):
for coords in coords:
self.all_nodes[int(coords[0])] = (coords[1], coords[2])
parser = FamilyParser()
p = OSMParser(concurrency = 2, ways_callback = parser.ways, nodes_callback = parser.nodes, coords_callback = parser.coords)
p.parse("%s/%s.osm.pbf" % (pbf_path, self.name))
for k in parser.nodes_objects.keys():
locations[k] = (parser.nodes_objects[k]["lon"], parser.nodes_objects[k]["lat"], parser.nodes_objects[k]["tag"])
for k in parser.ways_objects.keys():
# compute mean postion
m_lon, m_lat = 0, 0
for nd in parser.ways_objects[k]["nd"]:
node = parser.all_nodes[int(nd)]
m_lon += node[0]
m_lat += node[1]
m_lon /= float(len(parser.ways_objects[k]["nd"]))
m_lat /= float(len(parser.ways_objects[k]["nd"]))
locations[nd] = (m_lon, m_lat, parser.ways_objects[k]["tag"]) #a way can have the same id as a node so we take the id of the last node
logger.info("Number of elements in family %s : %s" % (family, str(len(locations.keys()))))
categories[family] = locations
for category in categories.keys():
for id in categories[category].keys():
building = Building(int(id), categories[category][id][0], categories[category][id][1], category, None)
self.features += [building]
def process_file(filename, baseurl = 'http://download.geofabrik.de/osm/europe/', parentCountry = ""):
print "retrieving file " + filename + "..."
urllib.urlretrieve(baseurl+filename, '/tmp/'+filename)
global country
refsSet = set()
nodesLookup = dict()
if parentCountry == "":
country = filename[0:-8].replace('_',' ').title()
if country in countriesLookup:
country = countriesLookup[country]
print "parsing for refs..."
parser = OSMParser(ways_callback=refs_handler)
parser.parse('/tmp/'+filename)
print "parsing for coords..."
parser = OSMParser(coords_callback=coords_handler)
parser.parse('/tmp/'+filename)
print "parsing for ways..."
parser = OSMParser(ways_callback=ways_handler)
parser.parse('/tmp/'+filename)
scraperwiki.sqlite.save_var(filename, 1)
def update_refs(filename):
# get refs
refs = GetRefs()
p = OSMParser(concurrency=4,
coords_callback=refs.coords)
p.parse(filename)
if len(Ref.objects.filter(need_update=True))>0:
current_task.retry(args=[filename])
return True
def reload_position_data(self):
print "Reloading (OSM) position data"
osm_stations = BusStationGetter()
p = OSMParser(concurrency=4, nodes_callback=osm_stations.nodes)
p.parse(OSM_file)
osm_stations = sorted(osm_stations.stations, key=itemgetter(0))
lpp_stations = sorted(get_lpp_stations(), key=itemgetter(0))
stations = get_coordinates_for_stations(lpp_stations, osm_stations)
pickle.dump(stations, open("lpp-station-geo-data.pypickle", "wb"))
print "Reloaded position data"
def add_churches(filename, invalidate_refs=True):
# get churches
churches = GetChurches(invalidate_refs=invalidate_refs)
p = OSMParser(concurrency=4,
nodes_callback=churches.nodes,
ways_callback=churches.ways)
p.parse(filename)
# don't run as task anymore:
# update_refs(filename)
return True
def reload_position_data(self):
print "Reloading (OSM) position data"
osm_stations = BusStationGetter()
p = OSMParser(concurrency=4, nodes_callback=osm_stations.nodes)
p.parse(OSM_file)
osm_stations = sorted(osm_stations.stations, key=itemgetter(0))
lpp_stations = sorted(get_lpp_stations(), key=itemgetter(0))
stations = get_coordinates_for_stations(lpp_stations, osm_stations)
pickle.dump(stations, open("lpp-station-geo-data.pypickle", "wb"))
print "Reloaded position data"
def parse(self):
parser = OSMParser(2,
nodes_callback=self.parse_nodes,
coords_callback=self.parse_coords,
ways_callback=self.parse_ways,
relations_callback=self.parse_relations,
nodes_tag_filter=self.nodes_filter,
ways_tag_filter=self.ways_filter,
relations_tag_filter=self.relations_filter,
)
osm_filename = os.path.join(os.path.dirname(__file__), self.osm_filename)
parser.parse(osm_filename)
def import_file(self, file_name):
"""Parses a file and feeds and imports the parsed osm primitives
into a remote crdb cluster
Parameters:
file_name -- the name of the input file
"""
parser = OSMParser(concurrency=10,
nodes_callback=self._on_nodes,
ways_callback=self._on_ways,
relations_callback=self._on_relations)
parser.parse(file_name)
def parse_osm_file(self):
osm_parser = OSMParser(
concurrency=2,
coords_callback=self.parse_points,
nodes_callback=self.parse_nodes,
ways_callback=self.parse_ways,
# relations_ callback=self.relations,
# nodes_tag_filter=self.nodes_filter,
# ways_tag_filter=self.ways_filter,
# relations_tag_filter=self.relations_filter
)
osm_parser.parse(self.osm_filename)
def __init__(self, osmfile) : # parse the input file and save its contents in memory # node ids and coord pairs as returned from imposm self.nodes = dict() # actual min and max latitude and longitude of coordinates self.bounds = dict() self.bounds["min_lat"] = 9999 self.bounds["max_lat"] = -9999 self.bounds["min_lng"] = 9999 self.bounds["max_lng"] = -9999 p = OSMParser(coords_callback = self.coords_callback) p.parse(osmfile)
def parse(fpath):
# instantiate counter and parser and start parsing
osm = Parser()
p = OSMParser(concurrency=2,
ways_callback=osm.ways_callback,
coords_callback=osm.coords_callback,
nodes_callback=osm.nodes_callback)
p.parse(fpath)
return osm
def init_mongo():
osm_parser = OSMParser(nodes_callback=save_places)
osm_parser.parse('belarus-latest.osm.pbf')
mongo_client = pymongo.MongoClient()
places_collection = mongo_client.map.places
places_collection.remove()
places_collection.create_index([('location', pymongo.GEOSPHERE)])
try:
places_collection.insert([p.as_dict() for p in places], continue_on_error=True)
except pymongo.errors.DuplicateKeyError:
pass
def test_save_bank(capfd):
"""Test saving a bank node
"""
# set up to parse `bank.osm`
input_filepath = os.path.join(os.path.dirname(__file__),"data","bank.osm")
node_handler = NodeHandler()
p = OSMParser(nodes_callback=node_handler.nodes)
p.parse(input_filepath)
# don't use db, so expect tab-separated STDOUT
resout, reserr = capfd.readouterr()
assert resout == '1\tBank of Testing\tbank\tPOINT(10.0 50.0)\n'
def processShed(pbfFile, outFile, distance, mode, concurrent, stopNodes):
# instantiate map and parser and start parsing
osmap = Map()
p = OSMParser(concurrency=concurrent, ways_callback=osmap.ways_callback, coords_callback=osmap.nodes_callback)
p.parse(pbfFile)
osmap.calculateWalkDistances(stopNodes)
walkshed = osmap.generateWalkShed(distance, {'distance': distance, 'mode': mode})
import json
with open(outFile, 'w') as f:
json.dump(walkshed, f)
def generateCache(filename = 'berlin.osm.pbf'):
cache = OSMCache('.')
coords_queue = JoinableQueue(512)
coords_writer = CacheWriterProcess(coords_queue, cache.coords_cache, None, marshaled_data=True)
coords_writer.start()
parser = OSMParser(coords_callback=coords_queue.put)
parser.parse(filename)
coords_queue.put(None)
coords_writer.join()
return cache
def handle(self, *args, **options):
"""
Handle the munin command
"""
fpath = sys.argv[2]
self.stdout.write("import {}\n".format(fpath))
# instantiate counter and parser and start parsing
chker = RosaParse()
p = OSMParser(concurrency=4, nodes_callback=chker.pnodes, relations_callback=chker.prelations)
# Read filename on cmd arg
p.parse(fpath)
def get_graph(filename):
grabber = OSMloader()
p = OSMParser(concurrency=1,
coords_callback=grabber.nodes,
ways_callback=grabber.ways)
p.parse(filename)
grabber.spinner.finish()
print ''
print 'Simplifying graph... '
graph = max(nx.strongly_connected_component_subgraphs(grabber.graph,
copy=True),
key=len)
return nx.DiGraph(simplify_graph(graph, strict=False))
def read(fname, bb=None, ll_bb=None):
g = graph.Graph()
vertex_map = {}
def coords_callback(clist):
for osmid, lon, lat in clist:
p = geom.FPoint(lon, lat)
if ll_bb is not None and not ll_bb.contains(p):
continue
p = coords.lonLatToMeters(p).to_point()
if bb is not None and not bb.contains(p):
continue
vertex_map[osmid] = g.add_vertex(p)
def ways_callback(ways):
for osmid, tags, refs in ways:
if 'highway' not in tags or tags['highway'] in BLACKLIST:
continue
for i in range(len(refs) - 1):
src = refs[i]
dst = refs[i + 1]
if src not in vertex_map or dst not in vertex_map:
continue
g.add_bidirectional_edge(vertex_map[src], vertex_map[dst])
p = OSMParser(coords_callback=coords_callback)
p.parse(fname)
p = OSMParser(ways_callback=ways_callback)
p.parse(fname)
return g
def parse_file(filename, scraper):
"""
parse_file
"""
print('Started parsing file: {}'.format(filename))
p = OSMParser(concurrency=4, nodes_callback=scraper.nodes)
start_time = time.clock()
p.parse(filename)
proc_time = time.clock()-start_time
# Add file to processed list, and update the progress file
scraper.add_file_processed(filename)
scraper.save_progress()
print(u"\n \u2713 Processed {} records in {} seconds".format(scraper.processed, proc_time))
def parse(self):
parser = OSMParser(
2,
nodes_callback=self.parse_nodes,
coords_callback=self.parse_coords,
ways_callback=self.parse_ways,
relations_callback=self.parse_relations,
nodes_tag_filter=self.nodes_filter,
ways_tag_filter=self.ways_filter,
relations_tag_filter=self.relations_filter,
)
osm_filename = os.path.join(os.path.dirname(__file__),
self.osm_filename)
parser.parse(osm_filename)
def update_refs(filename):
# get refs
missing_refs = Ref.objects.filter(need_update=True).count()
while missing_refs>0:
refs = GetRefs()
p = OSMParser(concurrency=4,
coords_callback=refs.coords)
p.parse(filename)
missing_refs_neu = Ref.objects.filter(need_update=True).count()
if missing_refs_neu == missing_refs:
return False
return True
def main():
file_path = sys.argv[1]
if not os.path.exists(file_path):
print("{} doesn't exist.".format(file_path))
exit(1)
else:
print("Extracting buildings from |{}|...".format(file_path))
root = et.Element('environment')
comment = et.Comment(
'Generated obstacle configurations for {}'.format(file_path))
root.append(comment)
coord_parser = CoordGeo()
p = OSMParser(concurrency=4,
coords_callback=coord_parser.coords_callback)
p.parse(file_path)
bldg_parser = BuildingShapes(coord_parser.coords)
p = OSMParser(concurrency=4, ways_callback=bldg_parser.ways_callback)
p.parse(file_path)
print('# Coords: {}'.format(len(coord_parser.coords)))
print('# Buildings: {}'.format(len(bldg_parser.xml_elmts)))
# Extent the children to the root element
root.extend(bldg_parser.xml_elmts)
#print(prettify(root))
# Dump to file
print('Writing result to {}'.format(output_file))
output = et.ElementTree(root)
output.write(output_file, pretty_print=True)
def __call__(self, filename):
client.init(splitext(basename(filename))[0], self.level)
if self.drop:
client.dropdb(self.level)
parser = OSMParser(concurrency=4,
ways_callback=self.way,
coords_callback=self.coord,
nodes_callback=self.node,
relations_callback=self.relation)
parser.parse(filename)
self.flush_buffer()
def retrieve_highway(osm_file, selected_zone, tolerance, Ncore=1):
# Parses the OSM file.
point = Point(selected_zone, tolerance)
p = OSMParser(concurrency=Ncore, coords_callback=point.select)
p.parse(osm_file)
highway = Highway(point)
p = OSMParser(concurrency=Ncore, ways_callback=highway.select)
p.parse(osm_file)
highway.point_set = set([item for refs in highway.point for item in refs])
# Getting all coordinates.
point_coordinate = []
for way in highway.point_set:
try:
point_coordinate.append(point.coordinate[way])
except:
pass
# Defining the set of OSM way id and coordinates.
highway_coordinate = []
highway_osmid = []
for refs, osmid in zip(highway.point, highway.osmid):
try:
highway_coordinate.append([point.coordinate[n] for n in refs])
highway_osmid.append(osmid)
except:
pass
return highway_coordinate, highway_osmid
def extract(self, pbf, output):
""" extract POI nodes from osm pbf extract """
logging.info("Extracting POI nodes from {0} to {1}".format(pbf, output))
with open(output, 'w') as f:
# define callback for each node that is processed
def nodes_callback(nodes):
for node in nodes:
node_id, tags, coordinates = node
# if any tags have a matching key then write record
if any([t in tags for t in POI_TAGS]):
f.write(json.dumps(dict(tags=tags, coordinates=coordinates)))
f.write('\n')
parser = OSMParser(concurrency=4, nodes_callback=nodes_callback)
parser.parse(pbf)
return output
def handle(self, *args, **options):
"""
Handle the munin command
"""
fpath = sys.argv[2]
self.stdout.write('import {}\n'.format(fpath))
# instantiate counter and parser and start parsing
chker = RosaParse()
p = OSMParser(concurrency=4,
nodes_callback=chker.pnodes,
relations_callback=chker.prelations)
# Read filename on cmd arg
p.parse(fpath)
class OSMPbfParserImpl:
def __init__(self,interface,osmPbfFile,callback_type=None,repository=None):
#print(IOSMPbfParser)#
if not isinstance(interface, IOSMPbfParser): raise Exception("Bad Interface")
self.osmPbfFile = osmPbfFile
if(callback_type== "nodes_callback"):
self.parser = OSMParser(concurrency=4,nodes_callback=self.fetchNodes)
print ("Initializing Nodes Callback......")
elif(callback_type== "ways_callback"):
self.parser = OSMParser(concurrency=4,ways_callback=self.fetchWays)
print ("Initializing Ways Callback......")
elif(callback_type== "relations_callback"):
self.parser = OSMParser(concurrency=4,relations_callback=self.fetchRelations)
print ("Initializing Relations Callback......")
elif(callback_type== "coords_callback"):
self.parser = OSMParser(concurrency=4,coords_callback=self.fetchCoords)
print ("Initializing Coords Callback......")
else:
self.parser = OSMParser(concurrency=4,nodes_callback=self.fetchNodes)
self.repository= repository
def start(self):
print ("Started Parser.......")
self.parser.parse(self.osmPbfFile)
def fetchNodes(self,nodes_tuple):
for osmId, meta_data,coords in nodes_tuple:
#print (osmId)
Graph.getInstance().addNode(osmId,Node(osmId,meta_data,coords))
#print(len(Graph.getInstance().getNodes()))
def fetchWays(self,ways_tuple,collection=None):
nodes = Graph.getInstance().getNodes().keys()
nodes_all = Graph.getInstance().getNodes()
for osmId, meta_data,refs in ways_tuple:
refs_set =set(refs)
#print ('Refs who are nodes')
common_nodes=list(refs_set.intersection(nodes))
if len(common_nodes)>1:
street_edges = []
for i in range(0,len(common_nodes)-1):
edge = Edge(Haversian.getInstance().calculateDistance(nodes_all[common_nodes[i]].getCoords(),nodes_all[common_nodes[i+1]].getCoords()),common_nodes[i],common_nodes[i+1])
#print('Haversian distance from '+str(common_nodes[i])+", "+str(common_nodes[i+1]))
#print(edge.getDistance())
street_edges.append(edge)
Graph.getInstance().addStreet(osmId, Street(osmId,meta_data,street_edges))
def fetchCoords(self,coords_tuple):
graph=Graph.getInstance()
for osmId,lng,lat in coords_tuple:
graph.addPoint(osmId, Point(lng,lat))
def generateCache(filename='berlin.osm.pbf'):
cache = OSMCache('.')
coords_queue = JoinableQueue(512)
coords_writer = CacheWriterProcess(coords_queue,
cache.coords_cache,
None,
marshaled_data=True)
coords_writer.start()
parser = OSMParser(coords_callback=coords_queue.put)
parser.parse(filename)
coords_queue.put(None)
coords_writer.join()
return cache
def main():
cli = axi.cli()
cli.add_argument("osm_file", help=".osm map file to read from")
args = cli.parse_args()
handler = Handler()
p = OSMParser(concurrency=1,
nodes_callback=handler.on_nodes,
ways_callback=handler.on_ways,
relations_callback=handler.on_relations,
coords_callback=handler.on_coords)
p.parse(args.osm_file)
projection = LambertAzimuthalEqualAreaProjection(LAT, LNG)
w = MAP_WIDTH_KM
#h = w * 12 / 8.5
h = w * args.height / args.width
gs = []
for key, ways in handler.ways.items():
if key not in WEIGHTS:
continue
weight = WEIGHTS[key]
paths = []
for way in ways:
coords = [projection.project(*handler.coords[x]) for x in way]
paths.append(coords)
g = paths_to_shapely(paths)
if weight:
g = g.buffer(LANE_WIDTH_M / 1000.0 * weight)
g = crop(g, w * MARGIN, h * MARGIN)
gs.append(g)
g = ops.cascaded_union(gs)
g = crop(g, w, h)
paths = shapely_to_paths(g)
#paths.append(circle(0, 0, 9 / 1000.0, 1000))
#paths.append(circle(0, 0, 6 / 1000.0, 1000))
#paths.append(circle(0, 0, 3 / 1000.0, 1000))
paths.append(box(w, h))
paths.append(box(w - 5.0 / 1000, h - 5.0 / 1000))
paths.append(box(w - 10.0 / 1000, h - 10.0 / 1000))
d = axi.Drawing(paths)
cli.draw(d)
def _buid_city(self):
city = City()
# Get origin of the map (we use the center)
tree = ET.parse(self.osm_map)
bounds = tree.find('bounds').attrib
self.bounds = {
'origin': LatLon(float(bounds['minlat']), float(bounds['minlon'])),
'corner': LatLon(float(bounds['maxlat']), float(bounds['maxlon']))
}
self.map_origin = self.bounds['origin'].midpoint(self.bounds['corner'])
logger.debug("Map Origin: {0}".format(self.map_origin))
# Calculate bounding box based on lat/lon
min_xy = self._translate_coords(self.bounds['origin'])
max_xy = self._translate_coords(self.bounds['corner'])
self.bounding_box = BoundingBox(min_xy, max_xy)
self.bounds['size'] = abs(min_xy) * 2
logger.debug("Bound box size {0}".format(self.bounds['size']))
logger.debug("Bounding box from {0} to {1}".format(
self.bounds['origin'], self.bounds['corner']))
# Parse OSM map
self.parser = OSMParser(coords_callback=self._get_coords,
ways_callback=self._get_ways)
self.parser.parse(self.osm_map)
logger.debug("Number of ways: {}".format(len(self.osm_ways)))
logger.debug("Number of coords: {}".format(len(self.osm_coords)))
# Ground plane
city.set_ground_plane(
GroundPlane(max(self.bounds['size'].x, self.bounds['size'].y),
Point(0, 0, 0)))
self._create_roads(city)
self._create_intersections(city)
# self._create_buildings(city)
city.trim_roads()
return city
def importdb(filename):
global buf, buffered_coords, buffered_nodes, pushed_coords, pushed_nodes
buf=[]
buffered_coords=set()
buffered_nodes=set()
pushed_coords=set()
pushed_nodes=set()
global waycount, nodecount, coordcount
nodecount = waycount = coordcount = 0
p = OSMParser(concurrency=4,
ways_callback=way,
coords_callback=coord,
nodes_callback=node,
relations_callback=relation)
p.parse(filename)
flush_buffer()
def station_nodes_from_nodes():
def imposm_extract_railway_station_nodes_from_nodes(nodes):
target_node_tags=["station"]
excluding_node_tags=["light_rail","subway"]
for osmid, tags, (lon,lat) in nodes:
if tags.get("railway","") in target_node_tags and tags.get("station","") not in excluding_node_tags:
station_dict[osmid]=[tags.get("name","N/A"),lat,lon]
station_dict_complete[osmid]=[tags, lat, lon]
OSMParser(concurrency=CONCURRENCYVAL, nodes_callback=imposm_extract_railway_station_nodes_from_nodes).parse(FOLDER+FILE)
def main(filename, output):
#cache = generateCache(filename)
cache = OSMCache('.')
coords_queue = JoinableQueue(512)
coords_writer = CacheWriterProcess(coords_queue,
cache.coords_cache,
None,
marshaled_data=True)
coords_writer.start()
handler = Handler(cache, output)
parser = OSMParser(coords_callback=coords_queue.put,
nodes_callback=handler.nodes_callback,
ways_callback=handler.ways_callback)
parser.parse(filename)
coords_queue.put(None)
coords_writer.join()
def __init__(self, path, offset):
self.__processor = DataProcessor()
self.__path = path
way = OSMParser(concurrency=4,
ways_callback=self.__processor.callback_ways)
coord = OSMParser(concurrency=4,
coords_callback=self.__processor.callback_coords)
way.parse(path)
coord.parse(path)
self.__structure = self.__processor.pre_processing(offset)
def extract(self, pbf, output):
""" extract POI nodes from osm pbf extract """
logging.info("Extracting POI nodes from {0} to {1}".format(
pbf, output))
with open(output, 'w') as f:
# define callback for each node that is processed
def nodes_callback(nodes):
for node in nodes:
node_id, tags, coordinates = node
# if any tags have a matching key then write record
if any([t in tags for t in POI_TAGS]):
f.write(
json.dumps(dict(tags=tags,
coordinates=coordinates)))
f.write('\n')
parser = OSMParser(concurrency=4, nodes_callback=nodes_callback)
parser.parse(pbf)
return output
def station_nodes_from_relations():
def imposm_extract_railway_stations_from_relations(relations):
'''get accurate railway stations from exact railways relations instead of getting wrong subway stations'''
target_relation_tags=["train","rail"]
for osmid, tags, members in relations:
if tags.get("route") in target_relation_tags or tags.get("railway") in target_relation_tags:
for i in members:
if i[1]=="node" and (i[0] in station_dict):
station_dict[i[0]].append("railwaysuffix")
OSMParser(concurrency=CONCURRENCYVAL, relations_callback=imposm_extract_railway_stations_from_relations).parse(FOLDER+FILE)
def __init__(self,interface,osmPbfFile,callback_type=None,repository=None):
#print(IOSMPbfParser)#
if not isinstance(interface, IOSMPbfParser): raise Exception("Bad Interface")
self.osmPbfFile = osmPbfFile
if(callback_type== "nodes_callback"):
self.parser = OSMParser(concurrency=4,nodes_callback=self.fetchNodes)
print ("Initializing Nodes Callback......")
elif(callback_type== "ways_callback"):
self.parser = OSMParser(concurrency=4,ways_callback=self.fetchWays)
print ("Initializing Ways Callback......")
elif(callback_type== "relations_callback"):
self.parser = OSMParser(concurrency=4,relations_callback=self.fetchRelations)
print ("Initializing Relations Callback......")
elif(callback_type== "coords_callback"):
self.parser = OSMParser(concurrency=4,coords_callback=self.fetchCoords)
print ("Initializing Coords Callback......")
else:
self.parser = OSMParser(concurrency=4,nodes_callback=self.fetchNodes)
self.repository= repository
def get_graph(filename):
node_coords = {}
edges = []
etags = {}
def node(id, *coords):
node_coords[id] = coords
def way(id, tags, refs):
is_way = False
if 'highway' in tags:
is_way = True
if 'busway' in tags:
is_way = True
if not is_way: return
if tags['highway'] == 'footway': return
if tags['highway'] == 'cycleway': return
if tags['highway'] == 'steps': return
for i in range(len(refs) - 1):
edge = refs[i], refs[i + 1]
etags[edge] = tags
edges.append(edge)
if not is_oneway(tags):
tags['oneway'] = 'yes'
way(None, tags, refs[::-1])
def nodes(nodes):
for args in nodes:
node(*args)
def ways(ways):
for args in ways:
way(*args)
parser = OSMParser(
ways_callback=ways,
coords_callback=nodes,
)
parser.parse(filename)
return node_coords, edges, etags
def station_nodes_from_ways():
def imposm_extract_railway_station_nodes_from_ways(ways):
'''get accurate railway stations from exact railways ways instead of getting wrong subway stations'''
target_way_tags=["rail"]
for osmid, tags, refs in ways:
if tags.get("railway","") in target_way_tags:
for i in refs:
if i in station_dict:
station_dict[i].append("railwaysuffix")
OSMParser(concurrency=CONCURRENCYVAL, ways_callback=imposm_extract_railway_station_nodes_from_ways).parse(FOLDER+FILE)
def collect_subway_nodes():
def imposm_extract_subway_nodes_from_relations(relations):
target_relation_tags=["subway","light_rail","tram"]
for osmid, tags, members in relations:
if tags.get("route","") in target_relation_tags or tags.get("railway") in target_relation_tags:
for m in members:
if m[1]=="node":
subway_nodes_complete.add(m[0])
if m[1]=="way":
subway_ways_from_relations.add(m[0])
def imposm_extract_subway_nodes_from_ways(ways):
target_way_tags=["subway","light_rail","tram"]
for osmid, tags, refs in ways:
if tags.get("railway","") in target_way_tags or osmid in subway_ways_from_relations:
for r in refs:
subway_nodes_complete.add(r)
OSMParser(concurrency=CONCURRENCYVAL, relations_callback=imposm_extract_subway_nodes_from_relations).parse(FOLDER+FILE)
OSMParser(concurrency=CONCURRENCYVAL, ways_callback=imposm_extract_subway_nodes_from_ways).parse(FOLDER+FILE)
def load_file(self, filename):
# Reinitialize if we have a new file
ways = []
coords = {}
num_coords = 0
num_ways = 0
# status output
if self.verbose:
sys.stderr.write("\nLoading ways, each '-' is 100 ways, each row is 10,000 ways\n")
p = OSMParser(ways_callback=self.ways_callback)
p.parse(filename)
# status output
if self.verbose:
sys.stderr.write("\n{} ways matched in {} {mem:.1f}MB memory used,\n{} coordinates will be loaded, each '.' is 1% complete\n".format(len(self.ways), filename, len(self.coords), mem=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1048576))
total = len(self.coords)
if total < 100:
self.coords_marker = 1
else:
self.coords_marker = round(total/100)
p = OSMParser(coords_callback=self.coords_callback)
p.parse(filename)
# status output
if self.verbose:
sys.stderr.write("\nCoordinates loaded {mem:.1f}MB memory used.".format(mem=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1048576))
sys.stderr.flush()
# Join routes end-to-end and add them to the way list.
self.join_ways()
# status output
if self.verbose:
sys.stderr.write("\nJoining complete. {mem:.1f}MB memory used.".format(mem=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1048576))
sys.stderr.write("\nCalculating curvature, each '.' is 1% complete\n")
sys.stderr.flush()
# Loop through the ways and calculate their curvature
start_time = time.time()
self.calculate()
# status output
if self.verbose:
sys.stderr.write("\nCalculation complete, {mem:.1f}MB memory used".format(mem=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1048576))
sys.stderr.write('\nCalculation completed in {time:.1f} seconds'.format(time=(time.time() - start_time)))
sys.stderr.flush()
