def get_waze_nodes(complete_waze, parsed_waze):
waze_coords = parsed_waze # waze link coord
waze_data = {}
points = []
waze_lat = []
waze_lon = []
waze_type = []
waze_speed = []
node = 0
for i in range(1, complete_waze + 1):
node = node + 1
waze_data[node] = {}
lat = waze_coords[str(i)]["location"][0]
lon = waze_coords[str(i)]["location"][1]
waze_lat.append(lat)
waze_lon.append(lon)
waze_type.append('WAZE')
points.append([lat, lon])
waze_data[node]['lat'] = lat
waze_data[node]['lon'] = lon
waze_data[node]['speed'] = parsed_waze[str(i)]['speed']
waze_data[node]['ele'] = '0'
waze_data[node]['type'] = 'WAZE'
print lat
print len(waze_data)
elevation = Elevation()
# print elevation.getElevations(points)
elevation_output = elevation.getElevations(points)
node = 0
for i in elevation_output:
node = node + 1
waze_data[node]['ele'] = i
map_functions.export_to_JSON(waze_data, 'waze_nodes.json')
map_functions.plot_coords(waze_lat, waze_lon, waze_type, 16,
'waze_nodes.html')
return waze_data
def main(gpxs, imgs):
# load all tracks
batch = []
tracks = []
for filename in gpxs:
print('Loading %s...' % filename)
try:
dom = DOM.parse(filename)
tracks.append(GeotagQuery(TrackPoints(dom)))
except Exception as e:
print(str(e))
print('Loaded %d tracks.' % len(tracks))
for img in imgs:
try:
exif = GExiv2.Metadata(img)
if tuple(exif.get_gps_info()) != (0., 0., 0.):
continue
print('No GPS info for %s... ' % os.path.basename(img), end='')
candidates = []
time = exif.get_date_time()
for track in tracks:
result = track(time)
if result is not None:
candidates.append(result)
assert(len(candidates) <= 1)
if len(candidates) == 0:
print('Not found.')
continue
candidate = candidates[0]
if candidate[2] is not None:
print(str(candidate))
exif.set_gps_info(candidate[1], candidate[0], candidate[2])
exif.save_file()
else:
print('(!) %f, %f' % (candidate[0], candidate[1]))
batch.append(GooglePt(img, exif, candidate[0], candidate[1]))
except Exception as e:
raise e
# print(str(e))
print('Missing %d altitude queries.' % len(batch))
elevapi = Elevation(API_KEY)
elevapi.run(batch)
Settings.init(configFile)
# open output file (if it doesn't already exists)
if os.path.exists(options.outFile):
sys.exit(
"Warning: file '" + options.outFile +
"' already exists, remove it or change the destination using the '--output' option.\n"
)
with codecs.open(options.outFile, 'w', 'utf-8') as outputFile:
# print headers and elevationGrid
elevation = None
if options.enable3D:
elevation = Elevation(Projection.getProjection(),
minlat=minlat,
minlon=minlon,
maxlat=maxlat,
maxlon=maxlon,
googleAPIKey=options.googleAPIKey)
print_header(outputFile,
minlat=minlat,
minlon=minlon,
maxlat=maxlat,
maxlon=maxlon,
elevation=elevation)
print(" * Elevation data acquired")
else:
print_header(outputFile,
minlat=minlat,
minlon=minlon,
maxlat=maxlat,
maxlon=maxlon)
class GeoPath():
"""Construct a path between two points in lat/lon/elev across the earth.
src -- source tuple (lon, lat, elev above ground)
dst -- dest tuple (lon, lat, elev above ground)
elev -- elevation provider, must have a lookup method that takes a location tuple
resolution -- step size to use in meters
"""
def __init__(self,
src=None,
dst=None,
path=None,
elev=None,
resolution=30,
async=False,
**kwargs):
self.geod = Geod(ellps='WGS84')
if elev == None:
self.elev = Elevation(**kwargs)
self.src = src
self.dst = dst
self.elev = elev
if path == None:
az1, az2, dist = self.geod.inv(src[0], src[1], dst[0], dst[1])
#print("az = {}, {} dist = {}".format(az1, az2, dist))
# xsteps = abs((src[0]-dst[0])/resolution)
# ysteps = abs((src[1]-dst[1])/resolution)
# steps = int(math.ceil(min(xsteps,ysteps)))
# xslope = (dst[0]-src[0])/steps
# yslope = (dst[1]-src[1])/steps
npts = int(dist / resolution)
#print("src={}, dst={}, npts={}".format(src, dst, npts))
if not async:
# first point is the source
self.path = [(src[0], src[1], self.elev.lookup(src))]
for lon, lat in self.geod.npts(src[0], src[1], dst[0], dst[1],
npts):
self.path.append((lon, lat, self.elev.lookup((lon, lat))))
# make the path
# for i in range(1,steps):
# x = src[0]+i*xslope
# y = src[1]+i*yslope
# add destn location and elev
self.path.append((dst[0], dst[1], self.elev.lookup(dst)))
else:
logging.info("geopath creating npts async.")
def worker(obj, npts):
obj.npts = obj.geod.npts(obj.src[0], obj.src[1],
obj.dst[0], obj.dst[1], npts)
def proc_worker(q, src, dst, npts, geod):
q.put(geod.npts(src[0], src[1], dst[0], dst[1], npts))
self.q = Queue()
self.p = Process(target=proc_worker,
args=(self.q, src, dst, npts, self.geod))
self.p.start()
#self.g = gevent.spawn(worker, self, npts)
#self.g.start()
logging.info("geopath created with async process.")
else:
self.path = path
self.src = path[0]
self.dst = path[-1]
def run(self):
# cnv = Converter(self.config, self.logger)
el = Elevation(self.config, self.logger)
import traceback
import inspect
import time
import pymongo
from shapely.geometry import mapping, shape, Point, Polygon, MultiPolygon, box
import os
from .utils.pointSampler import PopulationBasedPointSampler
from .utils.processSamplePoints import AnalyzePoints
from .utils.plotPointResults import PlotCoverage, PlotLoss, PlotContours
SRTM_PATH = os.path.join(os.getcwd(), "../SRTM_RAW")
elev = Elevation(srtm_path=SRTM_PATH, mongo_str=None)
bp_planitapi = Blueprint('planitapi',
__name__,
template_folder='templates',
static_folder='static')
bp_planitapi_app = None
@bp_planitapi.record
def record_app(setup_state):
global bp_planitapi_app
bp_planitapi_app = setup_state.app
logging.info("record flask app called for planitapi.")
def evaluate_points(q, num_base, basestations,
loss_threshold, pointdoc, bounds,
tx_height, rx_height,
run_id, itwomparam,
conn_str = 'mongodb://*****:*****@eg-mongodb.bucknell.edu/ym015',
srtm_path = None):
"called from multiprocessing"
pid = os.getpid()
# to save our results.
logging.info("{} connecting to {}".format(pid, conn_str))
connection = pymongo.MongoClient(conn_str)
db = connection.get_default_database()
elev = Elevation(srtm_path=srtm_path, mongo_str= None)
# for distance calc.
geod = Geod(ellps='WGS84')
if basestations!=None:
num_base = len(basestations)
logging.info("{} evaluating points for {} bs (tx {} rx {}) -- {} points".format(
pid, num_base, tx_height, rx_height, len(pointdoc)))
for d in pointdoc:
rdoc = { 'name': d['name'],
'point_docid': d['_id'],
'loss_threshold': loss_threshold,
'run': run_id,
'num_basestations': num_base,
'bounds': json_util.dumps(bounds),
'itwomparam': itwomparam.__dict__,
'tx_height': tx_height,
'rx_height': rx_height}
if basestations!= None:
rdoc['basestations'] = basestations
olddocs = db['POINTRESULTS'].find(rdoc)
dc = olddocs.count()
if dc > 1:
ods = [x['_id'] for x in olddocs]
logging.info("{} ({} bs) - {} has extra docs ({}), removing {}".format(d['name'], num_base, d['_id'], dc, ods[1:]))
db['POINTRESULTS'].remove({'_id': {'$in': ods[1:]}})
continue
if dc == 1:
logging.info("{} - {} done already".format(d['name'], d['_id']))
for doc in olddocs:
q.put(doc)
continue
logging.info("{} processing points for {} ({} bs) (tx {} rx {}) -- {} points".format(
pid, d['name'], num_base, tx_height, rx_height, len(pointdoc)))
# select base station point(s)
if basestations == None:
basestations = random.sample(d['points'], num_base)
# for b in basestations:
# if 'coordinates' not in b:
# b['coordinates'] = shape(b['geometry']).coords[0]
resultdoc = {
'name': d['name'],
'area': d['area'],
'population': d['population'],
'point_docid': d['_id'],
'run': run_id,
'loss_threshold': loss_threshold,
'num_basestations': num_base,
'num_points': len(d['points']),
'itwomparam': itwomparam.__dict__,
'bounds': json_util.dumps(bounds),
'nodes': [],
'grid': [],
'basestations': basestations,
'gentime': datetime.datetime.utcnow(),
'tx_height': tx_height,
'rx_height': rx_height
}
if 'state' in d:
resultdoc['state'] = d['state']
loss = []
# can't check this way we have multiple runs of the same point with diff. base locations.
# qry = {'point_docid': d['_id']}
# if db['POINTRESULTS'].find(qry).limit(1).count() > 0:
# print("already have results for {}".format(d['_id']))
# continue
connected = 0
disconnected = 0
# for b in basestations:
# if 'coordinates' not in b['geometry']:
# logging.info("no coordinates for {}".format(b))
#
# #convert gemoetry to geoJSON
#
# b['geometry'] = {'type': 'Point',
# 'coordinates': (b['geometry']['longitude'], b['geometry']['latitude'])}
#
# logging.info("added coordinates: {}".format(b['geometry']))
for p in d['points']:
if p in basestations:
#print("{} is a basestation".format(p))
continue
else:
# if 'coordinates' not in p:
# logging.info("no coordinates for {}".format(p))
# p['coordinates'] = shape(p['geometry']).coords[0]
# logging.info("added coordinates: {}".format(p['coordinates']))
#compute loss to all basestations, and save the lowest loss path
l = [(i,)+ point_loss(
(b['geometry']['coordinates'][0], b['geometry']['coordinates'][1]), tx_height,
(p['geometry']['coordinates'][0], p['geometry']['coordinates'][1]), rx_height,
params=itwomparam, elev=elev) for i,b in enumerate(basestations)]
logging.info("raw l is: {}".format(l))
l=sorted(l, key=lambda x:x[1])
logging.info("sorted l is: {}".format(l))
min_loss = l[0][1]
strmode = l[0][2]
loss.append(min_loss)
# az/distance to strongest base station
az12, az21, dist = geod.inv(
*basestations[l[0][0]]['geometry']['coordinates'],
*p['geometry']['coordinates']
)
logging.info("loss for {} to {} is: {}, min_loss: {}, distance: {}m".format(
", ".join([str(b['geometry']['coordinates']) for b in basestations]),
p['geometry']['coordinates'],
l, min_loss, dist
))
if min_loss < loss_threshold:
connected += 1
else:
disconnected += 1
resultdoc['nodes'].append(
{'point':p,
'loss': l,
'mode': strmode,
'bs_az': az12,
'pt_az': az21,
'distance': dist,
'min_loss': min_loss,
'connected': min_loss < loss_threshold})
if bounds != None:
# generate simple 10x10 grid around given bounds as well.
xmin,ymax,xmax,ymin = (bounds['west'], bounds['north'], bounds['east'], bounds['south'])
#g = np.mgrid[xmin:xmax:((xmax-xmin)/10), ymin:ymax:((ymax-ymin)/10)]
# use imaginary number to include bounds
# https://docs.scipy.org/doc/numpy/reference/generated/numpy.mgrid.html
g = np.mgrid[xmin:xmax:11j, ymin:ymax:11j]
# print("Generating points from bounds: {}".format(bounds))
# print("xmin,ymax,xmax,ymin: ", xmin,ymax,xmax,ymin)
for point in zip(*(x.flat for x in g)):
l = [point_loss(
(b['geometry']['coordinates'][0], b['geometry']['coordinates'][1]), tx_height,
(point[0], point[1]), rx_height,
params=itwomparam, elev=elev)[0] for b in basestations]
min_loss = min(l)
loss.append(min_loss)
resultdoc['grid'].append({
'point': {'geometry': mapping(Point(point))},
'loss': l,
'min_loss': min_loss
})
resultdoc['connected'] = connected / (connected+disconnected)
db['POINTRESULTS'].insert_one(resultdoc)
q.put(resultdoc)
logging.info("{} finished, close db.".format(pid))
connection.close()
logging.info("{} done.".format(pid))
'''
Created on Nov 29, 2014
@author: torsteinibo
'''
from lxml import etree
from mergeroads import nodes2nodeList
from elevation import Elevation
import sys
import utm
from math import fabs
import os
elevation = Elevation()
def getElevation(node):
lat = float(node.attrib['lat'])
lon = float(node.attrib['lon'])
res = utm.from_latlon(lat, lon, 33)
return elevation.getElevation(res[0], res[1])
def reverseWay(way, nd=None):
if nd is None:
nd = way.findall("nd")
for n in nd:
way.remove(n)
for i in range(len(nd) - 1, -1, -1):
way.append(nd[i])
db = connection.get_default_database()
#for statename in db['POINTRESULTS'].distinct('name'):
for statename in ["Pennsylvania"]:
outputfmt = "figures/states/{}_{}_{}.pdf"
ps = PopulationBasedPointSampler()
query = {'properties.NAME': statename}
state_shape = list(db['GENZ2010_040'].find(query))
query = {'state': '42', 'name': statename}
e = Elevation() # to look up point elevations.
plot_tracts = False
plot_elev = False
lossbynumbase = {}
loss_threshold = {}
cursor = db['POINTS'].find(query)
print("have {} points to process.".format(cursor.count()))
for i, pointdoc in enumerate(cursor):
print("{}: {} of {}.".format(statename, i, cursor.count()))
lqry = {'point_docid': pointdoc['_id']}
if i == 0:
for rsltdoc in db['POINTRESULTS'].find(lqry):
lossbynumbase[rsltdoc['num_basestations']] = [
if answer in ['y', 'n']:
answer = answer == 'y'
else:
answer = None
if not answer:
return
os.rename(filename, filename + '.bak')
with open(filename, 'w') as writer:
dom.writexml(writer)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Add elevation information to a pre-existing GPX track.')
parser.add_argument('gpx',
nargs='+',
help='GPX file(s) to modify (a backup will be made).')
parser.add_argument('-k', '--key', required=True, help='Google API key.')
args = parser.parse_args()
elevapi = Elevation(args.key)
for filename in args.gpx:
print('Processing %s...' % filename)
if not os.path.isfile(filename):
print('(E) Not a readable file.')
else:
try:
main(elevapi, filename)
except Exception as e:
print(e)
def plot_single_place(outputfmt, placename, cover_by_place_q):
#ps = PopulationBasedPointSampler()
connection = pymongo.MongoClient(
'mongodb://*****:*****@eg-mongodb.bucknell.edu/ym015')
db = connection.get_default_database()
e = Elevation() # to look up point elevations.
query = {'name': placename}
cover_by_place = []
lossbynumbase = {}
loss_threshold = {}
# qfilter = {'_id':1}
# if not os.path.exists(outputfmt.format(placename, i, 'elevation')):
# qfilter ['shapes'] = 1
# qfilter ['points'] = 1
# have to talk out the filter for just id to comptue other values.
cursor = db['POINTS'].find(query, {
'_id': 1,
'LSAD': 1,
'state': 1,
'points': 1,
'shapes': 1,
'shape_ids': 1
},
no_cursor_timeout=True) #.limit(2)
state = None
for i, pointdoc in enumerate(cursor):
state = pointdoc['state']
print("{}: {} of {}.".format(placename, i + 1, cursor.count()))
lqry = {'point_docid': pointdoc['_id'], 'tx_height': 5, 'rx_height': 1}
#lqry = {'point_docid': pointdoc['_id']}
if not os.path.exists(outputfmt.format(placename, i, 'loss')):
if i == 0:
for rsltdoc in db['POINTRESULTS'].find(lqry, {
'nodes': 1,
'num_basestations': 1,
'loss_threshold': 1
}):
lossbynumbase[rsltdoc['num_basestations']] = [
x['min_loss'] for x in rsltdoc['nodes']
if not math.isnan(x['min_loss'])
]
loss_threshold[rsltdoc['num_basestations']] = [
rsltdoc['loss_threshold']
]
else:
for rsltdoc in db['POINTRESULTS'].find(lqry, {
'nodes': 1,
'num_basestations': 1,
'loss_threshold': 1
}):
lossbynumbase[rsltdoc['num_basestations']] += [
x['min_loss'] for x in rsltdoc['nodes']
if not math.isnan(x['min_loss'])
]
loss_threshold[rsltdoc['num_basestations']].append(
rsltdoc['loss_threshold'])
#cqry = {'point_docid': pointdoc['_id'], 'tx_height': 5, 'rx_height': 1, 'num_basestations': 1}
cqry = {'point_docid': pointdoc['_id'], 'num_basestations': 1}
cover_by_place += [
x['connected']
for x in db['POINTRESULTS'].find(cqry, {'connected': 1})
]
#skip the rest, for now.
#continue
#print(lossbynumbase)
#print(loss_threshold)
# stats are the same, so only do it once.
if i == 0:
if (not os.path.exists(outputfmt.format(placename, i, 'elevation'))) or \
(not os.path.exists(outputfmt.format(placename, i, 'stats'))):
fig = plt.figure(figsize=(8, 8))
ax = plt.subplot(111)
if pointdoc['LSAD'] == 'County':
c_shp_query = {
'properties.NAME': placename,
'properties.STATE': pointdoc['state']
}
c_shp = list(db['GENZ2010_050'].find(c_shp_query))
#plot_shapes(ax, city_shp, filled= True, show_states = True, fc=lambda x:'red', alpha=0.99)
plot_shapes(ax, c_shp, filled=False, show_states=True)
elif pointdoc['LSAD'] == 'city':
city_shp_query = {
'properties.NAME': placename,
'properties.STATE': pointdoc['state']
}
city_shp = list(db['GENZ2010_160'].find(city_shp_query))
#plot_shapes(ax, city_shp, filled= True, show_states = True, fc=lambda x:'red', alpha=0.99)
plot_shapes(ax, city_shp, filled=False, show_states=True)
else:
print(
"WARN: unknown LSAD: {}, not plotting outline".format(
pointdoc['LSAD']))
if 'shapes' not in pointdoc:
print("getting shapes...")
sqry = {'_id': {'$in': pointdoc['shape_ids']}}
shapes = list(db['GENZ2010_140'].find(sqry))
else:
shapes = pointdoc['shapes']
plot_shapes(ax, shapes, filled=True, show_states=True)
# this is one way to do it
# tract_shp = list(ps.get_tract_shapes_in_area(city_shp[0]))
# plot_shapes(ax, tract_shp, filled= True, show_states = True)
elev_data = []
for pt in pointdoc['points']:
ax.plot(pt['coordinates'][0],
pt['coordinates'][1],
'.',
color='black',
ms=4,
lw=2,
alpha=0.75)
elev_data.append(e.lookup(pt['coordinates']))
ax.ticklabel_format(useOffset=False, style="plain")
ax.set_title("{}, {}".format(placename, state_name(state)))
ax.axis('equal')
plt.tight_layout()
plt.savefig(outputfmt.format(placename, i, 'tracts'))
plt.close(fig)
# show elevation distribution
fig = plt.figure(figsize=(8, 8))
ax = plt.subplot(111)
ax.hist(elev_data,
normed=1,
fill=False,
ec="black",
lw=3,
hatch=".")
ax.set_title(placename)
ax.set_xlabel('Elevation (meters)')
ax.set_ylabel('Rate')
ax.axis('tight')
plt.savefig(outputfmt.format(placename, i, 'elevation'))
plt.close(fig)
print("{} has {} tracts.".format(placename, len(shapes)))
areas = [
sh['properties']['area']['effective'] / (1000.0**2)
for sh in shapes if 'area' in sh['properties']
] # convert meters to KM
pops = [
sh['properties']['population']['effective'] / 1000.0
for sh in shapes if 'area' in sh['properties']
]
#print (areas)
#print (pops)
print('total pop: ', sum(pops), 'median: ', numpy.median(pops))
print('total area: ', sum(areas), 'median: ',
numpy.median(areas))
if not os.path.exists(outputfmt.format(placename, i, 'stats')):
if areas != None and pops != None:
fig = plt.figure(figsize=(8, 4))
ax = plt.subplot(121)
ax.hist(areas,
normed=0,
fill=False,
ec='black',
lw=3,
hatch='/')
ax.set_title("Area")
ax.set_xlabel('Square Kilometers')
ax.set_ylabel('Count')
ax.axis('normal')
ax = plt.subplot(122)
ax.set_title("Population")
ax.hist(pops,
normed=0,
fill=False,
ec='black',
lw=3,
hatch='x')
ax.set_xlabel('Population (Thousands)')
ax.set_ylabel('Count')
ax.axis('normal')
#ax.violinplot ([areas, pops], showmeans=False, showmedians=True)
#ax.set_xticklabels(['Area', 'Population'])
plt.tight_layout()
plt.savefig(outputfmt.format(placename, i, 'stats'))
plt.close(fig)
#if not os.path.exists(outputfmt.format(placename, i, 'loss')):
if 1:
# show loss distribution
fig = plt.figure(figsize=(8, 4))
ax = plt.subplot(111)
bases = sorted(lossbynumbase.keys())
if len(bases) > 0:
# print ('plot bases: ', end="")
# pprint(bases)
# print ('loss values: ', end="")
# pprint ([lossbynumbase[bs] for bs in bases])
ax.violinplot([lossbynumbase[bs] for bs in bases],
bases,
showmeans=False,
showmedians=True)
# print('medians are: ', end="")
# pprint([numpy.median(loss_threshold[i]) for i in bases])
ax.plot(bases, [numpy.median(loss_threshold[i]) for i in bases],
'--',
alpha=0.5,
color='green',
lw=2)
ax.set_title("{}, {}".format(placename, state_name(state)))
ax.set_xlabel('Number of basestations')
ax.set_ylabel('Loss (dB)')
ax.axis('tight')
plt.tight_layout()
plt.savefig(outputfmt.format(placename, i, 'loss'))
plt.close(fig)
else:
print(lossbynumbase)
print(bases)
cover_by_place_q.put((placename, cover_by_place))
"Warning: file '" + options.outFile +
"' already exists, remove it or change the destination using the '--output' option.\n"
)
sys.exit(0)
outputFile = codecs.open(options.outFile, 'w', 'utf-8')
# get settings from config file
Settings.init(options.configFile)
# print headers and elevationGrid
elevation = None
if options.enable3D:
elevation = Elevation(Projection.getProjection(),
minlat=minlat,
minlon=minlon,
maxlat=maxlat,
maxlon=maxlon)
print_header(outputFile,
minlat=minlat,
minlon=minlon,
maxlat=maxlat,
maxlon=maxlon,
elevation=elevation)
print(" * Elevation data acquired")
else:
print_header(outputFile,
minlat=minlat,
minlon=minlon,
maxlat=maxlat,
maxlon=maxlon)