#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import urllib.request
from xml.dom.minidom import parseString
import gpxpy.geo
import overpass
import overpy
from bs4 import BeautifulSoup
from Road import *
api = overpy.Overpass()
api2 = overpass.API()
def getNearestNode(listNode, coordinates):
distanceMin = 1000000000000000000000
latNearest = 0
lonNearest = 0
idNearest = 0
for i in range(len(listNode)):
lat = listNode[i].lat
lon = listNode[i].lon
id = listNode[i].id
distance = getDistance(lat, lon, coordinates['latitude'], coordinates['longitude'])
if distance < distanceMin:
import overpass
api = overpass.API()
response = api.get(
'https://data.osmbuildings.org/0.2/anonymous/tile/15/17605/10743.json')
#Useful functions @racu10
import pandas as pd
import numpy as np
import overpy
import overpass
import folium
from osmapi import OsmApi
import math
import geopy
import geopy.distance
import time
import unicodedata
MyApi = OsmApi()
apiOverPass = overpass.API()
apiOverPy = overpy.Overpass()
# In[2]:
def getDistance(long1, lat1, long2, lat2):
""" getDistance(long1, lat1, long2, lat2)
Get distance betwen 2 coordinates in log/lat.
Parameters
----------
long1 : float
Longitude 1st coordinate.
lat1 : float
from mapbox import Geocoder, Directions
import pickle
from shapely.geometry import Point, LineString
import time
import geopandas
import overpass
from shapely.geometry import LineString, Point
from oh_deer import config
year_file = open('oh_deer/static/yearly_trend_2010_2022.csv', 'r')
test = pd.read_csv(year_file, index_col=0)
test.index = pd.to_datetime(test.index)
#loaded_model = pickle.load(open("oh_deer/static/deer_pred.pickle.dat", "rb"))
loaded_model = pickle.load(open("oh_deer/static/pa_deer_pred.pickle.dat",
"rb"))
api_pass = overpass.API(timeout=600)
dummy_df = pd.read_csv('oh_deer/static/dummy_features.csv', index_col=0)
def deer_date(date):
"""
Given a date give a probability of seeing deer.
"""
def gaussian(x, height, center, width, offset):
return height * np.exp(-(x - center)**2 / (2 * width**2)) + offset
def three_gaussians(x, h1, c1, w1, h2, c2, w2, h3, c3, w3, offset):
# pip install overpass
import overpass
api = overpass.API(timeout=600, debug=True)
response = api.get(
'node["tourism"="alpine_hut"](41.836828,-5.009766,54.977614,29.267578);out 2;',
responseformat="geojson")
# filtering for 50 points only
responseNew = response[0:51]
print(responseNew)
const=None,
required=True,
help='File containing newline-delimited areas (OSM boundaries).')
parser.add_argument('--apikey',
action='store',
dest='apikey',
const=None,
required=True,
help='Google Static Maps API Key.')
args = parser.parse_args()
OUTPUT_DIR = "../data/bts_images/"
if __name__ == '__main__':
op = overpass.API(timeout=600)
with open(args.area) as t:
areas = t.readlines()
ls_areas = [a.strip() for a in areas]
for boundary in ls_areas:
print("Location: " + boundary + ". Searching for BTS...")
bts = op.Get('area(' + str(boundary) + '); \
(node["tower:type"="communication"]["communication:mobile_phone"="yes"](area);); \
out;')
bts_set = set(f.geometry.coordinates for f in bts.features)
path = OUTPUT_DIR + boundary + "/"
def generate(self):
DBJob.file_lock.acquire()
if not exists(self.sosm) or not exists(self.sdb):
if self.sdb not in DBJob.db_locks:
DBJob.db_locks[self.sdb] = Lock()
DBJob.file_lock.release()
print 'fetching {}...'.format(self.sosm)
boxlatlong = tileno2latlong(self.x, self.y, self.z)
if self.source in expandedsources:
expw = boxlatlong.width() / 50.0
exph = boxlatlong.height() / 50.0
else:
expw = 0.0
exph = 0.0
boxlatlong = Box2d(boxlatlong.minx - expw, boxlatlong.miny - exph,
boxlatlong.maxx + expw, boxlatlong.maxy + exph)
response = None
with DBJob.overpass_lock:
retry = 0
while retry < OSM_MAX_RETRIES:
try:
import overpass
api = overpass.API(
'http://overpass-api.de/api/interpreter',
timeout=None)
r = api._get_from_overpass('''
[out:json][bbox:{0},{1},{2},{3}];
{4}
out meta;
>;
out skel qt;
'''.format(boxlatlong.miny, boxlatlong.minx,
boxlatlong.maxy, boxlatlong.maxx,
datasources[self.source]))
r.encoding = 'UTF-8'
response = r.text
break
except overpass.MultipleRequestsError:
time.sleep(3 * retry)
retry += 1
if response is not None:
dirpath = dirname(self.sosm)
DBJob.dir_lock.acquire()
if not os.path.exists(dirpath):
os.makedirs(dirpath)
DBJob.dir_lock.release()
with open(self.sosm, 'wb') as w:
w.write(response.encode('utf8', 'replace'))
dirpath = dirname(self.sdb)
DBJob.dir_lock.acquire()
if not os.path.exists(dirpath):
os.makedirs(dirpath)
DBJob.dir_lock.release()
with DBJob.db_locks[self.sdb]:
toSpatial(self.sosm, self.sdb)
#with open(self.sosm, 'w') as w:
# w.write('')
else:
DBJob.file_lock.release()
self.doneEvent.set()
import overpass
import json
import os
path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)) )
api = overpass.API()
api = overpass.API(endpoint="https://overpass-turbo.eu")
api = overpass.API(timeout=25)
response = api.Get("""nwr(around:10000,45.4637901,9.1895511)["leisure"="park"];
out center;""")
with open(path+"/Park/Park.json", 'w') as outfile:
json.dump(response, outfile)
def apnd_from_overpass(gdfin,
querybound,
queryfeatures={'way': ['landuse', 'highway']},
values_in=None,
values_out=None,
CountValues=False):
"""
Get the queried features from the Overpass API
# for documentation see https://wiki.openstreetmap.org/wiki/Overpass_API
Currently supported for polygons only
values_in=either None or list of strings
values_out= either None or list of strings
"""
def divide_chunks(l, n):
# looping till length l
for i in range(0, len(l), n):
yield l[i:i + n]
# Connect to server
api = overpass.API(timeout=2000)
# get data from OSM and count server request try'"'
for element, _ in queryfeatures.items():
#if entry is string, we convert to list
if isinstance(queryfeatures[element], str):
queryfeatures[element] = [queryfeatures[element]]
for key in queryfeatures[element]:
# remove index_right if occuring
try:
gdfin.drop('index_right', axis=1)
except Exception:
pass
print('start to query data for osm element:', element, 'and key: ',
key, 'from overpass')
while True:
try:
osm_retrieved = api.get(element + '[' + key + '](' +
querybound + ');(._;>;);',
verbosity='geom')
except Exception as e:
print(e)
print('retry connection to server')
continue
break
# if no features were retrieved write nan and continue
if len(osm_retrieved['features']) == 0:
print(
'no data retrieved for osm type:',
element,
'and key ',
key,
)
gdfin[element[0] + '_' + key] = np.nan
continue
# convert to geodataframe, if to many information do it in chunks otherwise memory error may occur
chunk_length = 10000
osm_retrieved_chunks = list(
divide_chunks(osm_retrieved['features'], chunk_length))
gdfosm = gpd.GeoDataFrame()
def test_valid_geometry(f):
try:
shape(f)
valid_geo = True
except:
valid_geo = False
return valid_geo
for osm_retrieved_chunk in osm_retrieved_chunks:
#only consider elements with valid geometry
#for osm_retrieved in osm_retrieved_chunk:
#try:
# d = {"geometry": shape(osm_retrieved["geometry"]) if osm_retrieved["geometry"] else None}
#except:
# print('does not work')
#Linestrings need to have two points
osm_retrieved_chunk[:] = [
x for x in osm_retrieved_chunk
if test_valid_geometry(x["geometry"]) is True
]
#try:
# d = {"geometry": shape(osm_retrieved_chunks["geometry"]) if osm_retrieved_chunks["geometry"] else None}
#except:
# print('Problem appeared')
gdfosm_chunk = gpd.GeoDataFrame.from_features(
osm_retrieved_chunk, crs='epsg:4326')
if element == 'way':
# delete all points
gdfosm_chunk = gdfosm_chunk[
gdfosm_chunk.geom_type != "Point"]
# drop all columns except of the relevant key
retained_columns = ['geometry', key]
gdfosm_chunk.drop(
gdfosm_chunk.columns.difference(retained_columns),
1,
inplace=True)
# delete entries if defined in value_out, e.g the service
if values_out is not None:
if isinstance(values_out, str):
values_out = [values_out]
for value_out in values_out:
gdfosm_chunk = gdfosm_chunk[
gdfosm_chunk[key] != value_out]
# if value in is defined we look for this value only in the key
if values_in is not None:
if isinstance(values_in, str):
values_in = [values_in]
for value_in in values_in:
gdfosm_chunk = gdfosm_chunk[gdfosm_chunk[key] ==
value_in]
# in this case we have to rename the colfrom key to value
gdfosm_chunk = gdfosm_chunk.rename(
columns={key: value_in})
# bad solution...
#add together
gdfosm = gdfosm.append(gdfosm_chunk,
ignore_index=True,
sort=True)
#overwrite key and value_in if not None
if values_in is not None:
key = value_in
# add ID column to input data if not existing
if 'ID' not in gdfin.columns:
gdfin.insert(gdfin.shape[1], 'ID', range(0, gdfin.shape[0]))
# join with inout gdf
try:
gdfjoined = gpd.tools.sjoin(gdfin, gdfosm, how='left')
except Exception as e: # happens if all entrys are defined as values out
print(e)
gdfjoined = gdfin.copy()
gdfjoined[key] = None
# if the number of entries larger than the No of TrackPoints,
# we need to sum and find maxmimum
if gdfjoined.shape[0] > gdfin.shape[0]:
# group by unique ones https://stackoverflow.com/questions/
# 36174624/
s = gdfjoined.groupby(['ID', key]).size()
# check whether values of the key or count is needed
if CountValues:
dfselection = s.reset_index().drop(key, axis=1)
dfselection = dfselection.rename(
columns={0: 'No_of_' + key})
gdfmerged = gdfin.join(dfselection.set_index('ID'),
on='ID')
gdfmerged['No_of_' + key].fillna(0, inplace=True)
else:
dfselection = s.loc[s.groupby(
level=0).idxmax()].reset_index().drop(0, axis=1)
# replace values in the original dataset
# https://stackoverflow.com/questions/53262894/
gdfmerged = gdfin.join(dfselection.set_index('ID'),
on='ID')
else: # if it is equal things are more easy
if CountValues:
gdfjoined.rename(columns={key: 'No_of_' + key})
try:
gdfjoined['No_of_' + key].fillna(0, inplace=True)
except Exception:
print('No values type', key, 'queried')
gdfmerged = gdfjoined
# remove index_right if occuring
try:
gdfmerged.drop('index_right', axis=1, inplace=True)
except Exception:
pass
gdfin = gdfmerged.rename(columns={key: element[0] + '_' + key})
print('data for osm type:', element, 'and key ', key, 'retrieved')
gdfout = gdfin
return gdfout
def __init__(self):
self.api = overpass.API()
self.last_lat = -1
self.last_lon = -1
self.last_res = None
def get_nearest_waterways(gdf_from, id_col, max_distance=500, waterway_type='all'):
"""
Function to get the nearest waterways and associated nodes at some max distance from the OSM overpass server. Can specify the waterway type as either 'natural' or 'all'. The higher the max distance the slower the server query.
Parameters
----------
gdf_from : GeoDataFrame
Locations of the sites where you want to find the nearest waterway nodes.
id_col : str
The column name of the site ID for gdf_from.
max_distance : int
The maximum search distance.
waterway_type : str
Either 'nautral' or 'all'
Returns
-------
GeoDataFrame
of the sites and associated waterys and nodes.
"""
if waterway_type == 'all':
q_base = """(way['waterway'~'(river|stream|drain|canal|tidal_channel|ditch)'](around:{dis}, {lat}, {lon});
node(around:{dis}, {lat}, {lon})(w);)"""
elif waterway_type == 'natural':
q_base = "(way['waterway'~'(river|stream|tidal_channel)'](around:{dis}, {lat}, {lon}); node(around:{dis}, {lat}, {lon})(w);)"
else:
raise ValueError('waterway_type must be either natural or all.')
from1 = gdf_from[[id_col, 'geometry']].copy()
pts1 = from1.to_crs(4326)
from1['x'] = from1.geometry.x
from1['y'] = from1.geometry.y
from1['lon'] = pts1.geometry.x
from1['lat'] = pts1.geometry.y
api = overpass.API(endpoint=op_endpoint)
print('Overpass endpoint is: ' + op_endpoint)
no_node_ids = []
res_list = []
for index, p in from1.iterrows():
# print(p[id_col])
q1 = q_base.format(dis=max_distance, lat=p.lat, lon=p.lon)
response = api.get(q1, responseformat='json')
time.sleep(1)
nodes1 = [n for n in response['elements'] if n['type'] == 'node']
pd_nodes = pd.DataFrame.from_records(nodes1)
if pd_nodes.empty:
print('No node was found during the query for ' + id_col + ': ' + str(p[id_col]))
no_node_ids.extend([p[id_col]])
continue
gpd_nodes1 = gpd.GeoDataFrame(pd_nodes, geometry=gpd.points_from_xy(
pd_nodes['lon'], pd_nodes['lat']), crs=4326)
gpd_nodes2 = gpd_nodes1.to_crs(from1.crs)
nodes2 = list(zip(gpd_nodes2.geometry.x, gpd_nodes2.geometry.y))
btree = cKDTree(nodes2)
dist, idx = btree.query([p.x, p.y], k=1, distance_upper_bound=max_distance)
best1 = gpd_nodes2.iloc[[idx]].copy()
best1['distance'] = round(dist, 2)
best1[id_col] = p[id_col]
ways1 = [n for n in response['elements'] if n['type'] == 'way']
[s['tags'].update({'name': 'No name'}) for s in ways1 if not 'name' in s['tags']]
if len(ways1) == 1:
this_way = {'waterway_id': ways1[0]['id'], 'waterway_name': ways1[0]['tags']['name']}
else:
this_way = [{'waterway_id': w['id'], 'waterway_name': w['tags']['name']}
for w in ways1 if best1['id'].iloc[0] in w['nodes'][1:]][0]
best1['waterway_id'] = this_way['waterway_id']
best1['waterway_name'] = this_way['waterway_name']
res_list.append(best1)
res1 = pd.concat(res_list, sort=False).reset_index(drop=True)
return res1, no_node_ids
def getMaxspeed(coord):
'''
Recupera dalla API overpass la velocità massima vigente in un determinato punto, a partire dalle sue coordinate.
:type coord: string
:param coord: le coordinate del punto del quale si vuole scoprire la velocità massima vigente. Le coordinate sono in formato striga "latitudine,longitudine"
'''
api = overpass.API(endpoint="http://localhost/api/interpreter")
query = '(around:10,%s)' % (coord)
way_query = overpass.WayQuery(query)
response = api.Get(way_query)
results = response["features"]
roads = []
#primo filtraggio: per ogni coordinata assegno una o più strade, eliminando tutto il resto
for result in results:
if result["properties"]:
result = result["properties"]
if "highway" in result:
if "motorway" in result["highway"]: roads.append(result)
if "trunk" in result["highway"]: roads.append(result)
if "primary" in result["highway"]: roads.append(result)
if "secondary" in result["highway"]: roads.append(result)
if "tertiary" in result["highway"]: roads.append(result)
if "unclassified" in result["highway"]: roads.append(result)
if "residential" in result["highway"]: roads.append(result)
#se l'array roads non contiene nessuna strada allora non è stata trovata nessuna strada
if len(roads) == 0:
return -2
if len(roads) > 0:
roadmaxspeeds = []
for road in roads:
maxspeed = 0
#se c'è il tag "maxspeed" allora prendo direttamente quel valore
if "maxspeed" in road and road["maxspeed"].isdigit():
maxspeed = int(road["maxspeed"])
#se il tipo di "highway" è "residential", "trunk" o "motorway" allora non serve differenziare tra strada urbana o interurbana
if maxspeed == 0:
if "residential" in road["highway"]:
maxspeed = 50
if "trunk" in road["highway"]:
maxspeed = 90
if "motorway" in road["highway"]:
maxspeed = 130
#se gli if precedenti falliscono allora controllo se la strada è urbana o interurbana (tramite presenza o meno del marciapiede) e poi controllo il tipo di strada (primary, secondary,tertiary o unclassified)
if maxspeed == 0:
speeds = {
"urban": {
"primary": 50,
"secondary": 50,
"tertiary": 50,
"unclassified": 50
},
"interurban": {
"primary": 90,
"secondary": 90,
"tertiary": 90,
"unclassified": 70
}
}
type = None
if "sidewalk" in road: type = "urban"
else: type = "interurban"
if road["highway"] != "primary" and road[
"highway"] != "secondary" and road[
"highway"] != "tertiary" and road[
"highway"] != "unclassified":
maxspeed = -1
else:
maxspeed = speeds[type][road["highway"]]
#se nessuno dei filtraggi precedenti trovasse la velocità massima, segnalo il fallimento della ricerca della velocità massima
if maxspeed == 0:
maxspeed = -1
roadmaxspeeds.append(maxspeed)
return max(roadmaxspeeds)
else:
return -1 #velocità massima non trovata
def build_db(geoId):
api = overpass.API(timeout=10000)
# New Hampshire and some surrounding areas
BOUNDING_BOX = '43.12103377575541,-73.7237548828125,44.797428998555674,-69.378662109375'
# Boyne Highlands
# BOUNDING_BOX = '45,-85,46,-84'
if geoId:
BOUNDING_BOX = BB_LIST[int(geoId)]
# Snowbird/Alta ... Let's see.
# BOUNDING_BOX = '40.5, -111.74, 40.601, -11.58'
# Park City
# BOUNDING_BOX = '40.5, -111.62, 40.8, -111.244'
# Core of White Mountains
#BOUNDING_BOX = '44.00269350325321,-71.64871215820312,44.36067856998804,-71.12411499023438'
# All of NE US/CAN
#BOUNDING_BOX = '37.70120736474139,-81.7822265625,49.23912083246698,-64.40185546874999'
# response = api.Get('node["name"="Salt Lake City"]')
# response = api.Get('way["piste:type"~"downhill|yes"](44.00269350325321,-71.64871215820312,44.36067856998804,-71.12411499023438)')
print('getting trails')
trail_response = api.Get(build_query('way["piste:type"~"downhill|yes"](', BOUNDING_BOX), 'json')
print('getting lifts')
lift_response = api.Get(build_query('way["aerialway"~""](',BOUNDING_BOX), 'json')
print('getting ski areas')
ski_area_response = api.Get(build_query('way["landuse"~"winter_sports"](', BOUNDING_BOX), 'json')
ski_areas = []
trails = []
lifts = []
print('storing ski areas')
for sa in ski_area_response.features:
exists = SkiArea.query.filter_by(osm_id=sa['id'])
if exists.count():
#TODO: Allow ski area updates to process
continue
sam = SkiArea()
sam.name = sa['properties'].get('name')
if not sam.name:
continue
sam.osm_id = sa['id']
sam.boundary = str(geometry.Polygon(sa['geometry']['coordinates']))
ski_areas.append(sam)
db.session.add(sam)
db.session.commit()
print('storing trails')
for trail in trail_response.features:
if trail['properties'].get('name') is None:
continue
exists = Trail.query.filter_by(osm_id=trail['id'])
if exists.count():
new_trail = False
t = exists[0]
else:
new_trail = True
t= Trail()
t.name = trail['properties'].get('name')
t.difficulty = trail['properties'].get('piste:difficulty').lower() if 'piste:difficulty' in trail['properties'] else None
t.osm_id = trail['id']
t.path = str(geometry.LineString(trail['geometry']['coordinates']))
if new_trail:
db.session.add(t)
db.session.commit()
if not new_trail:
# Remaining steps only apply to new trails
continue
# Determine which ski area this trail is a part of.
# results = db.session.query(SkiArea).filter(SkiArea.boundary.ST_Contains(geoalchemy2.WKBElement(geometry.LineString(trail['geometry']['coordinates']))))
results = db.session.query(SkiArea).filter(SkiArea.boundary.ST_Intersects(t.path))
if results.count() > 1:
print('way ' + str(trail['id']) + ' is confused')
if results.count() > 0:
t.ski_area_id = results[0].id
continue
# if results.count() != 1:
results = find_nearest_ski_area(t, api)
if results:
t.ski_area_id = results
continue
# t.ski_area = sam
db.session.commit()
db.session.commit()
print('storing lifts')
for lift in lift_response.features:
exists = Lift.query.filter_by(osm_id=lift['id'])
is_update = exists.count()
if not is_update:
lt = Lift()
else:
lt = exists[0]
lt.name = lift['properties'].get('name')
lt.type = lift['properties'].get('aerialway')
lt.osm_id = lift['id']
lt.path = str(geometry.LineString(lift['geometry']['coordinates']))
lt.occupancy = lift['properties'].get('aerialway:occupancy', lift['properties'].get('capacity'))
if not is_update:
db.session.add(lt)
db.session.commit()
if is_update:
continue
results = db.session.query(SkiArea).filter(SkiArea.boundary.ST_Contains(lt.path))
if results.count() == 1:
lt.ski_area_id = results[0].id
continue
results = find_nearest_ski_area(lt, api)
if results:
lt.ski_area_id = results
db.session.commit()
# Clean up results.
# Cycle through unaffiliated lifts and trails. Attempt to affiliate.
# Continue to cycle until all lifts trails have affiliated or iterating isn't helping.
lifts = Lift.query.filter_by(ski_area_id = None)
num_unaffiliated_lifts = lifts.count()
trails = Trail.query.filter_by(ski_area_id = None)
num_unaffiliated_trails = trails.count()
while num_unaffiliated_lifts > 0 and num_unaffiliated_trails > 0:
print(num_unaffiliated_lifts)
for lt in lifts:
result = find_nearest_ski_area(lt)
if result:
lt.ski_area_id = result
db.session.commit()
for t in trails:
result = find_nearest_ski_area(t)
if result:
t.ski_area_id = result
db.session.commit()
lifts = Lift.query.filter_by(ski_area_id = None)
trails = Trail.query.filter_by(ski_area_id = None)
if lifts.count() == num_unaffiliated_lifts and trails.count() == num_unaffiliated_trails:
break
num_unaffiliated_lifts = lifts.count()
num_unaffiliated_trails = trails.count()
# Associates ski areas with states
# TODO: Rename all of this.
update_ski_areas.update()
print 'done'
def test_initialize_api():
api = overpass.API()
assert isinstance(api, overpass.API)
assert api.debug is False
def build_municipality_list(department):
"""Build municipality list
"""
api = overpass.API()
response = api.Get(
'''[out:json];
relation
[boundary="administrative"]
[admin_level=8]
["ref:INSEE"~"{}..."];
out geom qt;'''.format(department),
responseformat="json",
build=False,
)
features = []
txt_content = ''
for relation in response.get('elements'):
outer_ways = []
for member in relation.get('members'):
if member.get('role') == 'outer':
way = []
for point in member.get('geometry'):
way.append((point.get('lon'), point.get('lat')))
outer_ways.append(way)
tags = relation.get('tags')
insee = tags.get('ref:INSEE')
name = tags.get('name')
postcode = tags.get('addr:postcode')
municipality_border = Feature(properties={
'insee': insee,
'name': name,
'postcode': postcode,
'_storage_options': {
'color': color_for_insee(department, insee),
'weight': '1',
'fillOpacity': '0.5',
},
}, )
border = lines_to_polygon(outer_ways)
if not border:
logging.warning('{} does not have borders'.format(name))
else:
municipality_border.geometry = Polygon([border])
features.append(municipality_border)
txt_content += '{},{},{}\n'.format(insee, name, postcode)
# write municipality geojson
muni_geojson_path = path.join(BORDER_PATH, '{}.geojson'.format(insee))
with open(muni_geojson_path, 'w') as fd:
fd.write(geojson.dumps(municipality_border))
# write department geojson
dep_geojson_path = path.join(STATS_PATH, '{}.geojson'.format(department))
with open(dep_geojson_path, 'w') as fd:
fd.write(geojson.dumps(FeatureCollection(features)))
# write txt
txt_path = path.join(STATS_PATH, '{}-municipality.txt'.format(department))
with open(txt_path, 'w') as fd:
fd.write(txt_content)
