def search_along_path(path, last_path_pos, mlatlng0, mlatlng1):
Rough_dist_tolerate = 52
Rough_angle_tolerate = 80
mhead = get_bearing_latlng2(mlatlng0, mlatlng1)
mindist = 1e6
mindistline = 1e6
minangle = 1e6
for i in range(last_path_pos, len(path) - 1):
dic = path[i]
splatlng = [dic[KeyGPSLat], dic[KeyGPSLng]]
dist1 = get_dist_meters_latlng2(splatlng, mlatlng1)
if dist1 > 300: continue
dist12 = dist_point_to_line_of_2pts(splatlng, mlatlng0, mlatlng1)
if KeyGPSBearing in dic:
sphead = dic[KeyGPSBearing]
else:
''' find truly moved two pts to get heading'''
j = i + 1
sphead = None
while j < len(path) and get_dist_meters_latlng2(
splatlng, [path[j][KeyGPSLat], path[j][KeyGPSLng]]) < 5:
if iprint >= 3:
print(
"search_along_path find truly moved dist to get heading",
i, j)
if KeyGPSBearing in path[j] and sphead is None:
sphead = path[j][KeyGPSBearing]
j += 1
if j == len(path): j = i + 1
if sphead is None:
sphead = get_bearing_latlng2(
splatlng, [path[j][KeyGPSLat], path[j][KeyGPSLng]])
if mindist > dist1: mindist = dist1
if mindistline > dist12: mindistline = dist12
if min_angle_diff(sphead, mhead) < minangle:
minangle = min_angle_diff(sphead, mhead)
if dist1 < Rough_dist_tolerate and dist12 < Rough_dist_tolerate and min_angle_diff(
sphead, mhead) < Rough_angle_tolerate:
return i
if iprint >= 3:
print("search_along_path fail: mhead", mhead, "min dist", mindist,
"min distline", mindistline, "min angle d", minangle)
return -1
def get_bear_given_nid12(n1, n2, mm_nid2latlng):
latlng1 = mm_nid2latlng.get(n1)
if latlng1 is None:
try:
latlng1 = crawl_nid_to_latlng(n1)
except:
return None
latlng2 = mm_nid2latlng.get(n2)
if latlng2 is None:
try:
latlng2 = crawl_nid_to_latlng(n2)
except:
return None
return get_bearing_latlng2(latlng1, latlng2)
def get_pass_by_trips(AddTlsNetfile, TaxiStorageFile):
netObj = sumolib.net.readNet(AddTlsNetfile)
bbox = netObj.getBBoxXY()
minX, minY = bbox[0]
maxX, maxY = bbox[1]
minLon, minLat = netObj.convertXY2LonLat(minX, minY)
maxLon, maxLat = netObj.convertXY2LonLat(maxX, maxY)
bufD = 150. * 0.00001 # boundary shrink
minLon, minLat, maxLon, maxLat = minLon + bufD, minLat + bufD, maxLon - bufD, maxLat - bufD
print("osm bbox", minLat, minLon, maxLat, maxLon)
data = {}
cnt, thresh, badline = 0, 1, 0
afn = mypydir + mapFolder + os.sep + "taxi" + os.sep + 'taxi-pass-append.txt'
def _write(wstr):
with open(afn, 'a') as f:
f.write(wstr)
with open(TaxiStorageFile, "r") as f:
for l in f:
st = l.split('"')
assert len(st) <= 3, l
if len(st) > 1:
l = st[0] + st[2] # get rid of ',' in between ""
st = l.split(",")
try:
slat = float(st[hindex("Pickup Centroid Latitude")])
slng = float(st[hindex("Pickup Centroid Longitude")])
elat = float(st[hindex("Dropoff Centroid Latitude")])
elng = float(st[hindex("Dropoff Centroid Longitude")])
except:
badline += 1
continue
# init filter by min/max lat/lon
vertm = dist_point_to_line_of_2pts([minLat, minLon], [slat, slng],
[elat, elng])
vertx = dist_point_to_line_of_2pts([maxLat, maxLon], [slat, slng],
[elat, elng])
# too far away
if vertm > 4000 or vertx > 4000:
continue
# filter out inside bbox, already generated
if in_bbox_latlng(slat, slng, minLat, minLon, maxLat,
maxLon) and in_bbox_latlng(
elat, elng, minLat, minLon, maxLat, maxLon):
continue
# get route as list of latlngs
try:
res = requests.post(config.GreenRoute_IP_Port + '/html/route',
data={
'startLat': slat,
'startLng': slng,
'endLat': elat,
'endLng': elng
}).json()
except:
print("Err requests.post " + config.GreenRoute_IP_Port +
'/html/route')
continue
latlnglst = []
for latlngStr in str(res['gpstrace']).split("~|"):
tmp = latlngStr.split(",")
latlnglst.append([float(tmp[0]), float(tmp[1])])
# find 1st pt to be in bbox:
i = 0
while i < len(latlnglst):
lat, lng = latlnglst[i]
if in_bbox_latlng(lat, lng, minLat, minLon, maxLat, maxLon):
break
i += 1
starti = i
if starti == len(latlnglst):
continue
while i < len(latlnglst):
lat, lng = latlnglst[i]
if not in_bbox_latlng(lat, lng, minLat, minLon, maxLat,
maxLon):
break
i += 1
endi = i - 1
airDist = get_dist_meters_latlng2(latlnglst[starti],
latlnglst[endi])
if airDist < 1200:
continue # air-dist too small
slat, slng = latlnglst[starti]
elat, elng = latlnglst[endi]
if 1: # Note: duration/seconds/dist/miles info is invalid
try:
seconds = int(st[hindex("Trip Seconds")])
except:
seconds = 0.
airDist = get_dist_meters_latlng2([slat, slng], [elat, elng])
try:
miles = float(st[hindex("Trip Miles")])
except:
miles = 0.
if airDist < 10 or miles < 0.01 or seconds < 1:
continue
speed = miles * MetersPerMile / seconds
hd = get_bearing_latlng2([slat, slng], [elat, elng])
date = st[hindex("Trip Start Timestamp")].strip()
weekday = get_weekday_index_given_mdyhms12(
date) # 09/18/2013 07:45:00 AM
hour = get_hour_index_given_str(date)
minute = get_minute_given_str(date)
month, day, year = date.split(' ')[0].split('/')
month, day, year = int(month), int(day), int(year)
if year != 2017: continue
if month != 4: continue
if year not in data: data[year] = {}
if month not in data[year]: data[year][month] = []
x1, y1 = netObj.convertLonLat2XY(slng, slat)
x2, y2 = netObj.convertLonLat2XY(elng, elat)
dic = {
'lat1': slat,
'lng1': slng,
'lat2': elat,
'lng2': elng,
'weekday': weekday,
'speed': speed,
'x1': x1,
'y1': y1,
'x2': x2,
'y2': y2,
'hd': hd,
'year': year,
'month': month,
"day": day,
"hour": hour,
"minute": minute
}
data[year][month].append(dic)
_write(json.dumps(dic) + '\n')
cnt += 1
if cnt >= thresh:
thresh *= 2
print(cnt)
print(dic)
print("cnt", cnt, "badline", badline)
if cnt > 0:
fn = mypydir + mapFolder + os.sep + "taxi" + os.sep + 'taxi-pass.txt'
print("pickle dump to " + fn)
pickle.dump(data, open(fn, "wb"))
def get_google_latlngs_given_2latlng(latlng1,
latlng2,
overwrite=False,
show=False,
RouteStatsDir=None,
index=None,
html_show=None,
print_res=False):
key = "g%.5f,%.5f,%.5f,%.5f" % (latlng1[0], latlng1[1], latlng2[0],
latlng2[1])
if isinstance(index, int): index = str(index)
cache_file = _Cache_Dir + key
if not overwrite and os.path.exists(cache_file):
with open(cache_file, "r") as f:
print("read from cache " + cache_file)
return f.readline()
ret = gmaps.get_route_given_2latlng(
latlng1, latlng2) # [[latlng, dist, duration ],] sparse steps
# convert google's sparse latlng into dense latlng using osrm server.
latlngs = []
for stt in ret:
latlngs.append([stt[0][0], stt[0][1]])
nodeslist, addr = [], config.addr
get_fuel_given_latlng_list(latlngs,
addr,
addr2ip[addr],
URL_Match,
mm_nid2latlng,
loose_end_dist_allow=20,
nodeslist=nodeslist,
print_res=print_res)
if mm_nid2latlng.get_id() != "osm/cache-%s-nodeid-to-lat-lng.txt" % addr:
mm_nid2latlng.use_cache(
meta_file_name="osm/cache-%s-nodeid-to-lat-lng.txt" % addr,
ignore_invalid_mem=True)
# convert node ids into latlng list
ret = []
for nid in nodeslist:
latlng = mm_nid2latlng.get(nid)
if latlng is None: print("mm_nid2latlng get None", nid)
ret.append([latlng, 0, 0]) # same format [[latlng, dist, duration ],]
gps, distances, gmapTrace, sumDist = [], [], "", 0.
lastDx, lastDy = None, None
for i in range(len(ret) - 1):
latlng, dist, duration = ret[i]
latlng2, dist2, duration2 = ret[i + 1]
hd = get_bearing_latlng2(latlng, latlng2)
sumDist += get_dist_meters_latlng2(latlng, latlng2)
lng, lat = get_moved_xy(latlng[1], latlng[0], (hd + 90) % 360,
0.0000) # 0.0 means not moving
if lastDx is None:
lastDx, lastDy = (lng - latlng[1]), (lat - latlng[0])
else:
Dx, Dy = (lng - latlng[1]), (lat - latlng[0])
lng, lat = (Dx + lastDx) / 2. + latlng[1], (
Dy + lastDy) / 2. + latlng[0]
lastDx, lastDy = Dx, Dy
gps.append([lat, lng])
distances.append(dist)
gmapTrace += "new google.maps.LatLng(%.5f,%.5f)," % (lat, lng)
latlng, dist, duration = ret[-1]
lng, lat = get_moved_xy(latlng[1], latlng[0], (hd + 90) % 360, 0.0000)
Dx, Dy = (lng - latlng[1]), (lat - latlng[0])
lng, lat = (Dx + lastDx) / 2. + latlng[1], (Dy + lastDy) / 2. + latlng[0]
gps.append([lat, lng])
distances.append(dist)
strlist, gmapTrace, path = [], '', []
xs, ys = [], []
for i in range(len(gps) - 1):
latlng1, latlng2 = gps[i], gps[i + 1]
inter = interpolate_points(latlng1, latlng2,
max(2, int(distances[i + 1] / 30)))
for latlng in inter:
strlist.append("%.5f,%.5f" % (latlng[1], latlng[0]))
xs.append(latlng[1])
ys.append(latlng[0])
gmapTrace += "new google.maps.LatLng(%.5f,%.5f)," % (latlng[0],
latlng[1])
path.append(latlng)
strlist.append("%.5f,%.5f" % (latlng2[1], latlng2[0]))
gmapTrace += "new google.maps.LatLng(%.5f,%.5f)," % (latlng2[0],
latlng2[1])
path.append(latlng2)
if RouteStatsDir:
gen_map_html_from_path_list([path],
RouteStatsDir + "%s-gg.html" % index,
'',
disturb=False,
right_col_disp_list=[
html_show if html_show else "",
"Dist %.1f" % sumDist
])
xs.append(latlng2[1])
ys.append(latlng2[0])
if show:
plt.plot(xs, ys, '-ko')
plt.show()
wstr = " ".join(strlist)
if overwrite or not os.path.exists(cache_file):
with open(cache_file, "w") as f:
f.write(wstr)
return wstr
def parseSpeedCsvFile(SpeedSampleDatasetFile, netObj, pickleFile=None):
header = ["TIME","SEGMENT_ID","SPEED","STREET","DIRECTION","FROM_STREET","TO_STREET","LENGTH","STREET_HEADING","COMMENTS","BUS_COUNT","MESSAGE_COUNT","HOUR","DAY_OF_WEEK","MONTH","RECORD_ID","START_LATITUDE","START_LONGITUDE","END_LATITUDE","END_LONGITUDE","START_LOCATION","END_LOCATION","Community Areas","Zip Codes","Wards"]
bbox = netObj.getBBoxXY()
minX, minY = bbox[0]
maxX, maxY = bbox[1]
minLon, minLat = netObj.convertXY2LonLat(minX, minY)
maxLon, maxLat = netObj.convertXY2LonLat(maxX, maxY)
bufD = 300.*0.00001 # boundary shrink
moveD = 10.*0.00001 # move det location
LinesPerDay = 10000
minLon, minLat, maxLon, maxLat = minLon+bufD, minLat+bufD, maxLon-bufD, maxLat-bufD
print("osm bbox", minLat,minLon , maxLat,maxLon)
head = None
data = []
key2data={}
cnt , thresh , msg = 0,16,''
Data_minLon, Data_minLat, Data_maxLon, Data_maxLat = None,None,None,None
all_locs , loc_cnts = {} , {}
with open(SpeedSampleDatasetFile,"r") as f:
for l in f:
if head is None:
head=l
continue
cnt+=1
if cnt>= thresh:
print("l# %d, #data %d"%(cnt,len(data)))
thresh*=2
if cnt > 2000000 : # about enough?
if len(data)>0 and data[-1]['data'].has_enough_data() and data[0]['data'].has_enough_data():
msg = "Good amount of data, each weekday good"
break
if len(data)>5 and cnt > 4000000: # enough ?
msg = "Large amount of data, stop"
break
st = l.split(",")
if len(st)!=len(header):
print("bad line... "+l)
continue
elat = float( st[header.index("END_LATITUDE")] ) # look at end latlng
elon = float( st[header.index("END_LONGITUDE")] )
slat = float( st[header.index("START_LATITUDE")] ) # start loc gps
slon = float( st[header.index("START_LONGITUDE")] )
sname = st[header.index("STREET")].strip()
keyStr= "%s,%.5f,%.5f"%(sname,elat,elon)
loc_cnts[keyStr] = loc_cnts.get(keyStr,0)+1
if Data_minLat is None:
Data_minLon, Data_minLat, Data_maxLon, Data_maxLat = min(slon,elon), min(slat,elat), max(slon,elon), max(slat,elat)
if Data_minLon> min(slon,elon): Data_minLon = min(slon,elon)
if Data_minLat> min(slat,elat): Data_minLat = min(slat,elat)
if Data_maxLon< max(slon,elon): Data_maxLon = max(slon,elon)
if Data_maxLat< max(slat,elat): Data_maxLat = max(slat,elat)
speed = float(st[header.index("SPEED")])* constants.Mph2MetersPerSec
if speed>0 and in_bbox_latlng(elat,elon, minLat,minLon,maxLat,maxLon):
hd = get_bearing_latlng2([slat,slon], [ elat,elon])
direction = st[header.index("DIRECTION")].strip()
heading = get_heading_given_dirStr(direction)
if min_angle_diff(hd,heading)<50 :
dx, dy = slon-elon, slat - elat
dlon = dx / (dx**2+dy**2)**0.5 * moveD
dlat = dy / (dx**2+dy**2)**0.5 * moveD # move towards start a little
lon,lat = elon+dlon , elat+dlat
x,y = netObj.convertLonLat2XY(lon,lat)
name = st[header.index("STREET")].strip()
keyStr= "%s,%.5f,%.5f"%(name,lat,lon)
dateStr = st[header.index("TIME")] # 04/07/2019 02:40:36 PM
weekday = get_weekday_index_given_mdyhms12(dateStr)
hour = get_hour_index_given_str(dateStr)
mon, day, year = dateStr.split(" ",1)[0].split('/')
mon, day, year = int(mon), int(day), int(year)
if keyStr not in key2data:
sp = YearData(keyStr, 30)
sp.ingest(speed, hour, weekday, day, mon, year)
dic = { 'lat':lat, 'lng':lon, 'data': sp, 'x':x,'y':y ,"name":name, 'hd':hd, "dir":direction }
key2data[keyStr] = dic
data.append(dic)
else:
dic = key2data[keyStr]
dic['data'].ingest(speed, hour, weekday, day, mon, year)
# about 10,000 records per day
else:
print(l)
print("heading not matched...",hd,heading)
sys.exit(1)
pprint.pprint(data[0:5])
print("[parseSpeedCsvFile] len#",len(data)) # about 18 locations. (name,lat,lon)
print(msg)
print('#loc_cnts', len(loc_cnts))
return data
maxLon) and in_bbox_latlng(
elat, elng, minLat, minLon, maxLat, maxLon):
try:
seconds = int(st[ind("Trip Seconds")])
except:
seconds = 0.
airDist = get_dist_meters_latlng2([slat, slng], [elat, elng])
try:
miles = float(st[ind("Trip Miles")])
except:
miles = 0.
if airDist < 10 or miles < 0.01 or seconds < 1:
continue
speed = miles * MetersPerMile / seconds
hd = get_bearing_latlng2([slat, slng], [elat, elng])
date = st[ind("Trip Start Timestamp")].strip()
weekday = get_weekday_index_given_mdyhms12(
date) # 09/18/2013 07:45:00 AM
hour = get_hour_index_given_str(date)
minute = get_minute_given_str(date)
month, day, year = date.split(' ')[0].split('/')
month, day, year = int(month), int(day), int(year)
if year < minYear:
continue
if year not in data: data[year] = {}
if month not in data[year]: data[year][month] = []
x1, y1 = netObj.convertLonLat2XY(slng, slat)
x2, y2 = netObj.convertLonLat2XY(elng, elat)
''' ---- Find Start Position'''
matchPos = -1
startpos = None
while matchPos < len(nodeids) - 2:
matchPos += 1
if iprint >= 3:
print("Finding Start Position, try matchPos=%d" %
matchPos)
latlng0 = mm_nid2latlng.get(nodeids[matchPos])
latlng1 = mm_nid2latlng.get(nodeids[1 + matchPos])
if latlng0 is None or latlng1 is None:
print("mm_nid2latlng.get None:", nodeids[matchPos],
latlng0, nodeids[matchPos + 1], latlng1)
continue
hd1 = get_bearing_latlng2(latlng0, latlng1)
if iprint >= 3: print("head nodeids 0-1", hd1)
outsideBbox = 0
validStart = 0
for ind in range(len(path) - 1):
angleMatched = 0
distMatched = 0
dic = path[ind]
splat = dic[KeyGPSLat]
splng = dic[KeyGPSLng]
if not osm.within_bbox([splat, splng]):
outsideBbox = 1
break
sphead = None