class OverPassHelper:
fileName = "{objectType}_{area}.json"
filePath = None
defaultSelectors = [OsmDataQuery("streets", OsmType.WAY, ['"highway"'], "highway"),
OsmDataQuery("buildings", OsmType.WAY, ['"building"'], "building"),
OsmDataQuery("landuse", OsmType.WAY, ['"landuse"'], "landuse")]
def __init__(self, outPath='out/data/'):
# TODO: Validate path is directory
self.filePath = outPath + self.fileName
def getAreaId(self, locationName):
# TODO: check if its a place (otherwise following queries won't work)
nominatim = Nominatim()
return nominatim.query(locationName).areaId()
def getOsmGeoObjects(self, areaId, selector, elementType:OsmType):
"""
sends overpass-query and return the elements from the json response
"""
overpass = Overpass()
# out='geom' also leads to geometry key (list of coordinates for each object)
query = overpassQueryBuilder(
area=areaId, elementType=elementType.value, selector=selector, out='geom')
return overpass.query(query).toJSON()["elements"]
def saveGeoJson(self, file, data):
with open(file, 'w', encoding='UTF-8') as outfile:
# geojson.dump(data, outfile, ensure_ascii=False)
geojson.dump(data, outfile)
def fetch(self, areaId, areaName, osmQueries: List[OsmDataQuery] = None, overrideFiles=True) -> List[OsmDataQuery]:
""" fetch area data via overpassAPI and saves them as geojson
return a list of osmDataQuery where the filePath is set """
if not osmQueries:
osmQueries = self.defaultSelectors
for query in osmQueries:
file = self.filePath.format(objectType=query.name, area=areaName)
query.filePath = file
if Path(file).is_file() and not overrideFiles:
print("creation skipped, {} exists already".format(file))
else:
osmObjects = self.getOsmGeoObjects(areaId, query.osmSelector, query.osmObject)
print("Loaded {} {} for {}".format(
len(osmObjects), query.name, areaName))
geoJsonObjects = osmObjectsToGeoJSON(osmObjects)
self.saveGeoJson(file, geoJsonObjects)
return osmQueries
def directFetch(self, areaId, osmQueries = None) -> List:
"""returns list of geojson featurecollections"""
if isinstance(osmQueries, OsmDataQuery):
osmQueries = [osmQueries]
for query in osmQueries:
osmObjects = self.getOsmGeoObjects(areaId, query.osmSelector, query.osmObject)
yield osmObjectsToGeoJSON(osmObjects)
def timeMapsForPharmacies():
pharmacies = next(OverPassHelper().directFetch(dresdenAreaId, [OsmDataQuery("amenity health", OsmObjectType.ALL, ['"amenity"~"pharmacy"'])]))
timeMaps = retrieveTimeMaps(pharmacies["features"], travelTime=300, transportation="walking")
fileName = "out/data/timeMapsPerPharmacy.json"
with open(fileName, 'w', encoding='UTF-8') as outfile:
logging.info("Saving at {}".format(fileName))
geojson.dump(timeMaps, outfile)
def timeMapsForCityHalls():
townHalls = next(OverPassHelper().directFetch(dresdenAreaId, [OsmDataQuery("Town Halls", OsmObjectType.ALL, ['"amenity"="townhall"'])]))
# "driving+train" got many shapes (heatmap function could not handle them) .. trying "public_transport"
timeMaps = retrieveTimeMaps(townHalls["features"], travelTime=900, transportation="public_transport")
fileName = "out/data/timeMapsPerCityHall.json"
with open(fileName, 'w', encoding='UTF-8') as outfile:
logging.info("Saving at {}".format(fileName))
geojson.dump(timeMaps, outfile)
def getOpenAtMidnightThings():
thingsWithOpeningHour = next(OverPassHelper(
).directFetch(dresdenAreaId, [
OsmDataQuery("Midnight things", OsmObjectType.ALL, [
'"opening_hours"', '"highway"!~"."', '"tourism"!~"."',
'"leisure"!~"park|fitness_centre|bowling_alley|play_ground|playground"',
'"amenity"!~"parking|atm|hospital|charging_station|toilets|car_|vending_|bank|restaurant|bar$|pub$|nightclub|stripclub|brothel|cinema|theatre|drinking_water|nursing_home|recycling|shower|police|bicycle_|air|post_office"',
'"shop"!~"hairdresser"', '"office"!~"."'
])
]))
# general problem: opening hours must be filled in and valid & what should be excluded specifically
# TODO: easier to state what tags are allowed?
midnightThings = groupBy(
thingsWithOpeningHour,
lambda props: isOpenAtMidnight(props["opening_hours"]))
midnightThings.pop('False')
return midnightThings
def getPublicStops():
overpassFetcher = OverPassHelper()
pieschenAreaId = overpassFetcher.getAreaId("Dresden, Germany")
logging.info("Get stops from openstreetmap")
stopsOsmQuery = OsmDataQuery("Public Transport stops", OsmObjectType.NODE,
['"public_transport"="stop_position"'])
stops = next(overpassFetcher.directFetch(pieschenAreaId, [stopsOsmQuery]))
stopsByName = []
logging.info("Group stops by name")
for name, group in groupBy(stops, "name").items():
center = centerPoint(group)
# TODO: more properties?
# TODO: maybe draw line instead of just center point?
properties = {"name": name, "stop_positions": len(group["features"])}
stopByName = geojson.Feature(geometry=center, properties=properties)
stopsByName.append(stopByName)
return stopsByName
return stopsByName
if __name__ == "__main__":
COMPUTE_HEATMAPS = True
COMPUTE_VORONOI = True
map = Map(location=[51.078875, 13.728524],
tiles='Stamen Toner',
zoom_start=13)
TileLayer("openstreetmap").add_to(map)
logging.info("Get boundaries")
boroughs = next(
overpassFetcher.directFetch(dresdenAreaId, [
OsmDataQuery("Area boundaries", OsmObjectType.RELATIONSHIP,
['"boundary"~"administrative"', '"admin_level"="11"'])
]))
cityBoundary = next(
overpassFetcher.directFetch(saxonyAreaId, [
OsmDataQuery("Area boundaries", OsmObjectType.RELATIONSHIP,
['"boundary"~"administrative"', '"name"="Dresden"'])
]))
boundaries = {"boroughs": boroughs, "city": cityBoundary}
generateFeatureCollectionForGroups(boundaries, ["grey", "black"],
"Boundaries",
show=True).add_to(map)
pattern = "Public Transport (Pattern 16)"
sum([
entries
for type, entries in region["properties"]["safety"].items()
])
}
return (buildings, groups, regions)
logging.basicConfig(level=logging.INFO)
if __name__ == "__main__":
overPassFetcher = OverPassHelper()
areaOfInterest = 'Pieschen, Dresden, Germany'
pieschen = Nominatim().query(areaOfInterest)
allBuildingsQuery = OsmDataQuery("homes", OsmObjectType.WAYANDRELATIONSHIP,
['"building"', 'abandoned!~"yes"'])
osmQueries = [
allBuildingsQuery,
OsmDataQuery("borders", OsmObjectType.WAY, [
'highway~"primary|primary_link|secondary|secondary_link|tertiary|tertiary_link|residential|motorway|unclassified|living_street"'
]),
OsmDataQuery("borders_railway", OsmObjectType.WAY,
["'railway'~'rail'"])
]
# https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_polygon_.28poly.29 for filtering based on polygon (if borough based on openDataDresden)
# this query can take a while
osmData = overPassFetcher.directFetch(pieschen.areaId(), osmQueries)
buildings = next(osmData)
]
shopSelector = ['shop', 'leisure!~"."', 'building!~"."', 'amenity!~"."']
craftSelector = [
'craft', 'leisure!~"."', 'shop!~"."', 'building!~"."', 'amenity!~"."'
]
# add openDataDresden mapping (obsolete)
# objectTypeTagMapping = {
# "landnutzung1": "kuek_kl", "landnutzung2": "textstring"}
# openDataDresdenDir = "out/data/openDataDresden/"
# openDataDresdenFiles = {"landnutzung1": "{}nutzungsarten_1.geojson".format(
# openDataDresdenDir), "landnutzung2": "{}nutzungsarten_2.geojson".format(openDataDresdenDir)}
osmQueries = [
OsmDataQuery("streets", OsmObject.WAY, streetsSelector, "highway"),
OsmDataQuery("buildings", OsmObject.WAY, buildingsSelector, "building"),
OsmDataQuery("landuse", OsmObject.WAY, landuseSelector, "landuse"),
OsmDataQuery("railway", OsmObject.WAY, railwaySelector, "railway"),
OsmDataQuery("amenity", OsmObject.WAYANDNODE, amenitySelector, "amenity"),
OsmDataQuery("leisure", OsmObject.WAYANDNODE, leisureSelector, "leisure"),
OsmDataQuery("shop", OsmObject.NODE, shopSelector, "shop"),
OsmDataQuery("craft", OsmObject.NODE, craftSelector, "craft"),
]
osmDataFiles = OverPassHelper().fetch(pieschen.areaId(),
areaName,
osmQueries=osmQueries,
overrideFiles=True)
pieschenCoord = pieschen.toJSON()[0]
# handels register
file = open("out/data/scraper/handelsregister_Dresden_Pieschen.json",
encoding='UTF-8')
handelsRegisterCompanies = json.load(file)
registerFeature = geoFeatureCollectionToFoliumFeatureGroup(
handelsRegisterCompanies,
"blue",
"handelsRegister",
switchLatAndLong=False)
registerFeature.add_to(map)
# osm companies: often specify a name ? (tourism for holiday apartments f.i. , ... see data to collect)
namedAmenitiesThings = OsmDataQuery(
"osm_named_amenities", OsmObject.WAYANDNODE,
["name", "amenity", '"amenity"!~"vending_machine|parking"'], "")
namedLeisureThings = OsmDataQuery("osm_named_leisure", OsmObject.WAYANDNODE,
["name", "leisure", 'amenity!~"."'], "")
namedShopsThings = OsmDataQuery(
"osm_named_shops", OsmObject.WAYANDNODE,
["name", "shop", 'amenity!~"."', 'leisure!~"."'], "")
namedCraftThings = OsmDataQuery(
"osm_named_crafts", OsmObject.WAYANDNODE,
["name", "craft", 'amenity!~"."', 'leisure!~"."', 'shop!~"."'], "")
namedCompaniesThings = OsmDataQuery(
"osm_named_companies", OsmObject.WAYANDNODE, [
"name", "company", 'amenity!~"."', 'leisure!~"."', 'shop!~"."',
'craft!~"."'
], "")
osmQueries = [
import json
from OSMPythonTools.nominatim import Nominatim
import folium
from folium.plugins.measure_control import MeasureControl
import sys, os
sys.path.insert(1, os.path.abspath('..'))
from helper.geoJsonHelper import groupBy
from helper.overPassHelper import OverPassHelper
from helper.OsmDataQuery import OsmDataQuery
from helper.OsmObjectType import OsmObjectType
pieschen = Nominatim().query('Pieschen, Dresden, Germany')
osmQuery = OsmDataQuery("streets", OsmObjectType.WAY, ["'highway'"], "highway")
all_streets = next(OverPassHelper().directFetch(pieschen.areaId(), [osmQuery]))
street_types = groupBy(all_streets, ["highway"])
piescheonCoord = [pieschen.toJSON()[0]["lat"], pieschen.toJSON()[0]["lon"]]
streetMap = folium.Map(location=piescheonCoord,
tiles='Stamen Toner',
zoom_start=15)
icon = folium.Icon(icon="pills", color='lightgray', icon_color='red')
folium.Marker(location=piescheonCoord, tooltip="test",
icon=icon).add_to(streetMap)
for type, streets in street_types.items():
properties = list(streets["features"][0]["properties"].keys())
folium.GeoJson(