#!/usr/bin/env python # -*- coding: utf-8 -*- import os import urllib.request from xml.dom.minidom import parseString import gpxpy.geo import overpass import overpy from bs4 import BeautifulSoup from Road import * api = overpy.Overpass() api2 = overpass.API() def getNearestNode(listNode, coordinates): distanceMin = 1000000000000000000000 latNearest = 0 lonNearest = 0 idNearest = 0 for i in range(len(listNode)): lat = listNode[i].lat lon = listNode[i].lon id = listNode[i].id distance = getDistance(lat, lon, coordinates['latitude'], coordinates['longitude']) if distance < distanceMin:
import overpass api = overpass.API() response = api.get( 'https://data.osmbuildings.org/0.2/anonymous/tile/15/17605/10743.json')
#Useful functions @racu10 import pandas as pd import numpy as np import overpy import overpass import folium from osmapi import OsmApi import math import geopy import geopy.distance import time import unicodedata MyApi = OsmApi() apiOverPass = overpass.API() apiOverPy = overpy.Overpass() # In[2]: def getDistance(long1, lat1, long2, lat2): """ getDistance(long1, lat1, long2, lat2) Get distance betwen 2 coordinates in log/lat. Parameters ---------- long1 : float Longitude 1st coordinate. lat1 : float
from mapbox import Geocoder, Directions import pickle from shapely.geometry import Point, LineString import time import geopandas import overpass from shapely.geometry import LineString, Point from oh_deer import config year_file = open('oh_deer/static/yearly_trend_2010_2022.csv', 'r') test = pd.read_csv(year_file, index_col=0) test.index = pd.to_datetime(test.index) #loaded_model = pickle.load(open("oh_deer/static/deer_pred.pickle.dat", "rb")) loaded_model = pickle.load(open("oh_deer/static/pa_deer_pred.pickle.dat", "rb")) api_pass = overpass.API(timeout=600) dummy_df = pd.read_csv('oh_deer/static/dummy_features.csv', index_col=0) def deer_date(date): """ Given a date give a probability of seeing deer. """ def gaussian(x, height, center, width, offset): return height * np.exp(-(x - center)**2 / (2 * width**2)) + offset def three_gaussians(x, h1, c1, w1, h2, c2, w2, h3, c3, w3, offset):
# pip install overpass import overpass api = overpass.API(timeout=600, debug=True) response = api.get( 'node["tourism"="alpine_hut"](41.836828,-5.009766,54.977614,29.267578);out 2;', responseformat="geojson") # filtering for 50 points only responseNew = response[0:51] print(responseNew)
const=None, required=True, help='File containing newline-delimited areas (OSM boundaries).') parser.add_argument('--apikey', action='store', dest='apikey', const=None, required=True, help='Google Static Maps API Key.') args = parser.parse_args() OUTPUT_DIR = "../data/bts_images/" if __name__ == '__main__': op = overpass.API(timeout=600) with open(args.area) as t: areas = t.readlines() ls_areas = [a.strip() for a in areas] for boundary in ls_areas: print("Location: " + boundary + ". Searching for BTS...") bts = op.Get('area(' + str(boundary) + '); \ (node["tower:type"="communication"]["communication:mobile_phone"="yes"](area);); \ out;') bts_set = set(f.geometry.coordinates for f in bts.features) path = OUTPUT_DIR + boundary + "/"
def generate(self): DBJob.file_lock.acquire() if not exists(self.sosm) or not exists(self.sdb): if self.sdb not in DBJob.db_locks: DBJob.db_locks[self.sdb] = Lock() DBJob.file_lock.release() print 'fetching {}...'.format(self.sosm) boxlatlong = tileno2latlong(self.x, self.y, self.z) if self.source in expandedsources: expw = boxlatlong.width() / 50.0 exph = boxlatlong.height() / 50.0 else: expw = 0.0 exph = 0.0 boxlatlong = Box2d(boxlatlong.minx - expw, boxlatlong.miny - exph, boxlatlong.maxx + expw, boxlatlong.maxy + exph) response = None with DBJob.overpass_lock: retry = 0 while retry < OSM_MAX_RETRIES: try: import overpass api = overpass.API( 'http://overpass-api.de/api/interpreter', timeout=None) r = api._get_from_overpass(''' [out:json][bbox:{0},{1},{2},{3}]; {4} out meta; >; out skel qt; '''.format(boxlatlong.miny, boxlatlong.minx, boxlatlong.maxy, boxlatlong.maxx, datasources[self.source])) r.encoding = 'UTF-8' response = r.text break except overpass.MultipleRequestsError: time.sleep(3 * retry) retry += 1 if response is not None: dirpath = dirname(self.sosm) DBJob.dir_lock.acquire() if not os.path.exists(dirpath): os.makedirs(dirpath) DBJob.dir_lock.release() with open(self.sosm, 'wb') as w: w.write(response.encode('utf8', 'replace')) dirpath = dirname(self.sdb) DBJob.dir_lock.acquire() if not os.path.exists(dirpath): os.makedirs(dirpath) DBJob.dir_lock.release() with DBJob.db_locks[self.sdb]: toSpatial(self.sosm, self.sdb) #with open(self.sosm, 'w') as w: # w.write('') else: DBJob.file_lock.release() self.doneEvent.set()
import overpass import json import os path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)) ) api = overpass.API() api = overpass.API(endpoint="https://overpass-turbo.eu") api = overpass.API(timeout=25) response = api.Get("""nwr(around:10000,45.4637901,9.1895511)["leisure"="park"]; out center;""") with open(path+"/Park/Park.json", 'w') as outfile: json.dump(response, outfile)
def apnd_from_overpass(gdfin, querybound, queryfeatures={'way': ['landuse', 'highway']}, values_in=None, values_out=None, CountValues=False): """ Get the queried features from the Overpass API # for documentation see https://wiki.openstreetmap.org/wiki/Overpass_API Currently supported for polygons only values_in=either None or list of strings values_out= either None or list of strings """ def divide_chunks(l, n): # looping till length l for i in range(0, len(l), n): yield l[i:i + n] # Connect to server api = overpass.API(timeout=2000) # get data from OSM and count server request try'"' for element, _ in queryfeatures.items(): #if entry is string, we convert to list if isinstance(queryfeatures[element], str): queryfeatures[element] = [queryfeatures[element]] for key in queryfeatures[element]: # remove index_right if occuring try: gdfin.drop('index_right', axis=1) except Exception: pass print('start to query data for osm element:', element, 'and key: ', key, 'from overpass') while True: try: osm_retrieved = api.get(element + '[' + key + '](' + querybound + ');(._;>;);', verbosity='geom') except Exception as e: print(e) print('retry connection to server') continue break # if no features were retrieved write nan and continue if len(osm_retrieved['features']) == 0: print( 'no data retrieved for osm type:', element, 'and key ', key, ) gdfin[element[0] + '_' + key] = np.nan continue # convert to geodataframe, if to many information do it in chunks otherwise memory error may occur chunk_length = 10000 osm_retrieved_chunks = list( divide_chunks(osm_retrieved['features'], chunk_length)) gdfosm = gpd.GeoDataFrame() def test_valid_geometry(f): try: shape(f) valid_geo = True except: valid_geo = False return valid_geo for osm_retrieved_chunk in osm_retrieved_chunks: #only consider elements with valid geometry #for osm_retrieved in osm_retrieved_chunk: #try: # d = {"geometry": shape(osm_retrieved["geometry"]) if osm_retrieved["geometry"] else None} #except: # print('does not work') #Linestrings need to have two points osm_retrieved_chunk[:] = [ x for x in osm_retrieved_chunk if test_valid_geometry(x["geometry"]) is True ] #try: # d = {"geometry": shape(osm_retrieved_chunks["geometry"]) if osm_retrieved_chunks["geometry"] else None} #except: # print('Problem appeared') gdfosm_chunk = gpd.GeoDataFrame.from_features( osm_retrieved_chunk, crs='epsg:4326') if element == 'way': # delete all points gdfosm_chunk = gdfosm_chunk[ gdfosm_chunk.geom_type != "Point"] # drop all columns except of the relevant key retained_columns = ['geometry', key] gdfosm_chunk.drop( gdfosm_chunk.columns.difference(retained_columns), 1, inplace=True) # delete entries if defined in value_out, e.g the service if values_out is not None: if isinstance(values_out, str): values_out = [values_out] for value_out in values_out: gdfosm_chunk = gdfosm_chunk[ gdfosm_chunk[key] != value_out] # if value in is defined we look for this value only in the key if values_in is not None: if isinstance(values_in, str): values_in = [values_in] for value_in in values_in: gdfosm_chunk = gdfosm_chunk[gdfosm_chunk[key] == value_in] # in this case we have to rename the colfrom key to value gdfosm_chunk = gdfosm_chunk.rename( columns={key: value_in}) # bad solution... #add together gdfosm = gdfosm.append(gdfosm_chunk, ignore_index=True, sort=True) #overwrite key and value_in if not None if values_in is not None: key = value_in # add ID column to input data if not existing if 'ID' not in gdfin.columns: gdfin.insert(gdfin.shape[1], 'ID', range(0, gdfin.shape[0])) # join with inout gdf try: gdfjoined = gpd.tools.sjoin(gdfin, gdfosm, how='left') except Exception as e: # happens if all entrys are defined as values out print(e) gdfjoined = gdfin.copy() gdfjoined[key] = None # if the number of entries larger than the No of TrackPoints, # we need to sum and find maxmimum if gdfjoined.shape[0] > gdfin.shape[0]: # group by unique ones https://stackoverflow.com/questions/ # 36174624/ s = gdfjoined.groupby(['ID', key]).size() # check whether values of the key or count is needed if CountValues: dfselection = s.reset_index().drop(key, axis=1) dfselection = dfselection.rename( columns={0: 'No_of_' + key}) gdfmerged = gdfin.join(dfselection.set_index('ID'), on='ID') gdfmerged['No_of_' + key].fillna(0, inplace=True) else: dfselection = s.loc[s.groupby( level=0).idxmax()].reset_index().drop(0, axis=1) # replace values in the original dataset # https://stackoverflow.com/questions/53262894/ gdfmerged = gdfin.join(dfselection.set_index('ID'), on='ID') else: # if it is equal things are more easy if CountValues: gdfjoined.rename(columns={key: 'No_of_' + key}) try: gdfjoined['No_of_' + key].fillna(0, inplace=True) except Exception: print('No values type', key, 'queried') gdfmerged = gdfjoined # remove index_right if occuring try: gdfmerged.drop('index_right', axis=1, inplace=True) except Exception: pass gdfin = gdfmerged.rename(columns={key: element[0] + '_' + key}) print('data for osm type:', element, 'and key ', key, 'retrieved') gdfout = gdfin return gdfout
def __init__(self): self.api = overpass.API() self.last_lat = -1 self.last_lon = -1 self.last_res = None
def get_nearest_waterways(gdf_from, id_col, max_distance=500, waterway_type='all'): """ Function to get the nearest waterways and associated nodes at some max distance from the OSM overpass server. Can specify the waterway type as either 'natural' or 'all'. The higher the max distance the slower the server query. Parameters ---------- gdf_from : GeoDataFrame Locations of the sites where you want to find the nearest waterway nodes. id_col : str The column name of the site ID for gdf_from. max_distance : int The maximum search distance. waterway_type : str Either 'nautral' or 'all' Returns ------- GeoDataFrame of the sites and associated waterys and nodes. """ if waterway_type == 'all': q_base = """(way['waterway'~'(river|stream|drain|canal|tidal_channel|ditch)'](around:{dis}, {lat}, {lon}); node(around:{dis}, {lat}, {lon})(w);)""" elif waterway_type == 'natural': q_base = "(way['waterway'~'(river|stream|tidal_channel)'](around:{dis}, {lat}, {lon}); node(around:{dis}, {lat}, {lon})(w);)" else: raise ValueError('waterway_type must be either natural or all.') from1 = gdf_from[[id_col, 'geometry']].copy() pts1 = from1.to_crs(4326) from1['x'] = from1.geometry.x from1['y'] = from1.geometry.y from1['lon'] = pts1.geometry.x from1['lat'] = pts1.geometry.y api = overpass.API(endpoint=op_endpoint) print('Overpass endpoint is: ' + op_endpoint) no_node_ids = [] res_list = [] for index, p in from1.iterrows(): # print(p[id_col]) q1 = q_base.format(dis=max_distance, lat=p.lat, lon=p.lon) response = api.get(q1, responseformat='json') time.sleep(1) nodes1 = [n for n in response['elements'] if n['type'] == 'node'] pd_nodes = pd.DataFrame.from_records(nodes1) if pd_nodes.empty: print('No node was found during the query for ' + id_col + ': ' + str(p[id_col])) no_node_ids.extend([p[id_col]]) continue gpd_nodes1 = gpd.GeoDataFrame(pd_nodes, geometry=gpd.points_from_xy( pd_nodes['lon'], pd_nodes['lat']), crs=4326) gpd_nodes2 = gpd_nodes1.to_crs(from1.crs) nodes2 = list(zip(gpd_nodes2.geometry.x, gpd_nodes2.geometry.y)) btree = cKDTree(nodes2) dist, idx = btree.query([p.x, p.y], k=1, distance_upper_bound=max_distance) best1 = gpd_nodes2.iloc[[idx]].copy() best1['distance'] = round(dist, 2) best1[id_col] = p[id_col] ways1 = [n for n in response['elements'] if n['type'] == 'way'] [s['tags'].update({'name': 'No name'}) for s in ways1 if not 'name' in s['tags']] if len(ways1) == 1: this_way = {'waterway_id': ways1[0]['id'], 'waterway_name': ways1[0]['tags']['name']} else: this_way = [{'waterway_id': w['id'], 'waterway_name': w['tags']['name']} for w in ways1 if best1['id'].iloc[0] in w['nodes'][1:]][0] best1['waterway_id'] = this_way['waterway_id'] best1['waterway_name'] = this_way['waterway_name'] res_list.append(best1) res1 = pd.concat(res_list, sort=False).reset_index(drop=True) return res1, no_node_ids
def getMaxspeed(coord): ''' Recupera dalla API overpass la velocità massima vigente in un determinato punto, a partire dalle sue coordinate. :type coord: string :param coord: le coordinate del punto del quale si vuole scoprire la velocità massima vigente. Le coordinate sono in formato striga "latitudine,longitudine" ''' api = overpass.API(endpoint="http://localhost/api/interpreter") query = '(around:10,%s)' % (coord) way_query = overpass.WayQuery(query) response = api.Get(way_query) results = response["features"] roads = [] #primo filtraggio: per ogni coordinata assegno una o più strade, eliminando tutto il resto for result in results: if result["properties"]: result = result["properties"] if "highway" in result: if "motorway" in result["highway"]: roads.append(result) if "trunk" in result["highway"]: roads.append(result) if "primary" in result["highway"]: roads.append(result) if "secondary" in result["highway"]: roads.append(result) if "tertiary" in result["highway"]: roads.append(result) if "unclassified" in result["highway"]: roads.append(result) if "residential" in result["highway"]: roads.append(result) #se l'array roads non contiene nessuna strada allora non è stata trovata nessuna strada if len(roads) == 0: return -2 if len(roads) > 0: roadmaxspeeds = [] for road in roads: maxspeed = 0 #se c'è il tag "maxspeed" allora prendo direttamente quel valore if "maxspeed" in road and road["maxspeed"].isdigit(): maxspeed = int(road["maxspeed"]) #se il tipo di "highway" è "residential", "trunk" o "motorway" allora non serve differenziare tra strada urbana o interurbana if maxspeed == 0: if "residential" in road["highway"]: maxspeed = 50 if "trunk" in road["highway"]: maxspeed = 90 if "motorway" in road["highway"]: maxspeed = 130 #se gli if precedenti falliscono allora controllo se la strada è urbana o interurbana (tramite presenza o meno del marciapiede) e poi controllo il tipo di strada (primary, secondary,tertiary o unclassified) if maxspeed == 0: speeds = { "urban": { "primary": 50, "secondary": 50, "tertiary": 50, "unclassified": 50 }, "interurban": { "primary": 90, "secondary": 90, "tertiary": 90, "unclassified": 70 } } type = None if "sidewalk" in road: type = "urban" else: type = "interurban" if road["highway"] != "primary" and road[ "highway"] != "secondary" and road[ "highway"] != "tertiary" and road[ "highway"] != "unclassified": maxspeed = -1 else: maxspeed = speeds[type][road["highway"]] #se nessuno dei filtraggi precedenti trovasse la velocità massima, segnalo il fallimento della ricerca della velocità massima if maxspeed == 0: maxspeed = -1 roadmaxspeeds.append(maxspeed) return max(roadmaxspeeds) else: return -1 #velocità massima non trovata
def build_db(geoId): api = overpass.API(timeout=10000) # New Hampshire and some surrounding areas BOUNDING_BOX = '43.12103377575541,-73.7237548828125,44.797428998555674,-69.378662109375' # Boyne Highlands # BOUNDING_BOX = '45,-85,46,-84' if geoId: BOUNDING_BOX = BB_LIST[int(geoId)] # Snowbird/Alta ... Let's see. # BOUNDING_BOX = '40.5, -111.74, 40.601, -11.58' # Park City # BOUNDING_BOX = '40.5, -111.62, 40.8, -111.244' # Core of White Mountains #BOUNDING_BOX = '44.00269350325321,-71.64871215820312,44.36067856998804,-71.12411499023438' # All of NE US/CAN #BOUNDING_BOX = '37.70120736474139,-81.7822265625,49.23912083246698,-64.40185546874999' # response = api.Get('node["name"="Salt Lake City"]') # response = api.Get('way["piste:type"~"downhill|yes"](44.00269350325321,-71.64871215820312,44.36067856998804,-71.12411499023438)') print('getting trails') trail_response = api.Get(build_query('way["piste:type"~"downhill|yes"](', BOUNDING_BOX), 'json') print('getting lifts') lift_response = api.Get(build_query('way["aerialway"~""](',BOUNDING_BOX), 'json') print('getting ski areas') ski_area_response = api.Get(build_query('way["landuse"~"winter_sports"](', BOUNDING_BOX), 'json') ski_areas = [] trails = [] lifts = [] print('storing ski areas') for sa in ski_area_response.features: exists = SkiArea.query.filter_by(osm_id=sa['id']) if exists.count(): #TODO: Allow ski area updates to process continue sam = SkiArea() sam.name = sa['properties'].get('name') if not sam.name: continue sam.osm_id = sa['id'] sam.boundary = str(geometry.Polygon(sa['geometry']['coordinates'])) ski_areas.append(sam) db.session.add(sam) db.session.commit() print('storing trails') for trail in trail_response.features: if trail['properties'].get('name') is None: continue exists = Trail.query.filter_by(osm_id=trail['id']) if exists.count(): new_trail = False t = exists[0] else: new_trail = True t= Trail() t.name = trail['properties'].get('name') t.difficulty = trail['properties'].get('piste:difficulty').lower() if 'piste:difficulty' in trail['properties'] else None t.osm_id = trail['id'] t.path = str(geometry.LineString(trail['geometry']['coordinates'])) if new_trail: db.session.add(t) db.session.commit() if not new_trail: # Remaining steps only apply to new trails continue # Determine which ski area this trail is a part of. # results = db.session.query(SkiArea).filter(SkiArea.boundary.ST_Contains(geoalchemy2.WKBElement(geometry.LineString(trail['geometry']['coordinates'])))) results = db.session.query(SkiArea).filter(SkiArea.boundary.ST_Intersects(t.path)) if results.count() > 1: print('way ' + str(trail['id']) + ' is confused') if results.count() > 0: t.ski_area_id = results[0].id continue # if results.count() != 1: results = find_nearest_ski_area(t, api) if results: t.ski_area_id = results continue # t.ski_area = sam db.session.commit() db.session.commit() print('storing lifts') for lift in lift_response.features: exists = Lift.query.filter_by(osm_id=lift['id']) is_update = exists.count() if not is_update: lt = Lift() else: lt = exists[0] lt.name = lift['properties'].get('name') lt.type = lift['properties'].get('aerialway') lt.osm_id = lift['id'] lt.path = str(geometry.LineString(lift['geometry']['coordinates'])) lt.occupancy = lift['properties'].get('aerialway:occupancy', lift['properties'].get('capacity')) if not is_update: db.session.add(lt) db.session.commit() if is_update: continue results = db.session.query(SkiArea).filter(SkiArea.boundary.ST_Contains(lt.path)) if results.count() == 1: lt.ski_area_id = results[0].id continue results = find_nearest_ski_area(lt, api) if results: lt.ski_area_id = results db.session.commit() # Clean up results. # Cycle through unaffiliated lifts and trails. Attempt to affiliate. # Continue to cycle until all lifts trails have affiliated or iterating isn't helping. lifts = Lift.query.filter_by(ski_area_id = None) num_unaffiliated_lifts = lifts.count() trails = Trail.query.filter_by(ski_area_id = None) num_unaffiliated_trails = trails.count() while num_unaffiliated_lifts > 0 and num_unaffiliated_trails > 0: print(num_unaffiliated_lifts) for lt in lifts: result = find_nearest_ski_area(lt) if result: lt.ski_area_id = result db.session.commit() for t in trails: result = find_nearest_ski_area(t) if result: t.ski_area_id = result db.session.commit() lifts = Lift.query.filter_by(ski_area_id = None) trails = Trail.query.filter_by(ski_area_id = None) if lifts.count() == num_unaffiliated_lifts and trails.count() == num_unaffiliated_trails: break num_unaffiliated_lifts = lifts.count() num_unaffiliated_trails = trails.count() # Associates ski areas with states # TODO: Rename all of this. update_ski_areas.update() print 'done'
def test_initialize_api(): api = overpass.API() assert isinstance(api, overpass.API) assert api.debug is False
def build_municipality_list(department): """Build municipality list """ api = overpass.API() response = api.Get( '''[out:json]; relation [boundary="administrative"] [admin_level=8] ["ref:INSEE"~"{}..."]; out geom qt;'''.format(department), responseformat="json", build=False, ) features = [] txt_content = '' for relation in response.get('elements'): outer_ways = [] for member in relation.get('members'): if member.get('role') == 'outer': way = [] for point in member.get('geometry'): way.append((point.get('lon'), point.get('lat'))) outer_ways.append(way) tags = relation.get('tags') insee = tags.get('ref:INSEE') name = tags.get('name') postcode = tags.get('addr:postcode') municipality_border = Feature(properties={ 'insee': insee, 'name': name, 'postcode': postcode, '_storage_options': { 'color': color_for_insee(department, insee), 'weight': '1', 'fillOpacity': '0.5', }, }, ) border = lines_to_polygon(outer_ways) if not border: logging.warning('{} does not have borders'.format(name)) else: municipality_border.geometry = Polygon([border]) features.append(municipality_border) txt_content += '{},{},{}\n'.format(insee, name, postcode) # write municipality geojson muni_geojson_path = path.join(BORDER_PATH, '{}.geojson'.format(insee)) with open(muni_geojson_path, 'w') as fd: fd.write(geojson.dumps(municipality_border)) # write department geojson dep_geojson_path = path.join(STATS_PATH, '{}.geojson'.format(department)) with open(dep_geojson_path, 'w') as fd: fd.write(geojson.dumps(FeatureCollection(features))) # write txt txt_path = path.join(STATS_PATH, '{}-municipality.txt'.format(department)) with open(txt_path, 'w') as fd: fd.write(txt_content)