def format(self, record):
message = {'message': record.getMessage(),
'asctime': self.formatTime(record, self.datefmt),
'name': record.name,
'msg': record.msg,
'args': record.args,
'levelname': record.levelname,
'levelno': record.levelno,
'pathname': record.pathname,
'filename': record.filename,
'module': record.module,
'lineno': record.lineno,
'funcname': record.funcName,
'created': record.created,
'msecs': record.msecs,
'relative_created': record.relativeCreated,
'thread': record.thread,
'thread_name': record.threadName,
'process_name': record.processName,
'process': record.process,
'traceback': None}
if hasattr(record, 'extra'):
message['extra'] = record.extra
if record.exc_info:
message['traceback'] = self.formatException(record.exc_info)
return jsonutils.dumps(message)
def getCacheKey(self, params):
''' get cache key
@param params: params in dictionary
@return: key as string
'''
params['class'] = self.__class__.__name__
key = jsonutils.dumps(params)
print 'cache key:', key
return key
def validate(self, orig, dest):
''' validate route calculation with google map service
@param orig: start point in format {
'lat' : lat
'lng' : lng
}
@param dest: end point in format {
'lat' : lat
'lng' : lng
}
'''
p0 = LatLon(orig['lat'], orig['lon'])
p1 = LatLon(dest['lat'], dest['lon'])
# Telenav
tn_toll_route = self.ms.parseDirectionsResponse(
self.ms.directions(p0, p1, useCache=False))
tn_no_toll_route = self.ms.parseDirectionsResponse(
self.ms.directions(p0, p1, True, useCache=False))
#print 'status:', tn_no_toll_route.status
print 'distance(avoidtoll[tn]):', tn_no_toll_route.distance
#print 'time:', tn_no_toll_route.travel_time
#print
#print 'json:', tn_no_toll_route.rawdata
# Google
g_toll_route = self.googlems.parseDirectionsResponse(
self.googlems.directions(p0, p1))
g_no_toll_route = self.googlems.parseDirectionsResponse(
self.googlems.directions(p0, p1, True))
#print 'status:', g_no_toll_route.status
print 'distance(avoidtoll[google]):', g_no_toll_route.distance
#print 'time:', g_no_toll_route.travel_time
#print
#print 'json:', g_no_toll_route.rawdata
print 'd(t)/d(g):', float(
tn_no_toll_route.distance) / g_no_toll_route.distance
# d = orig2dest
d = {'orig': orig, 'dest': dest}
d['telenav_distance_notoll'] = tn_no_toll_route.distance
d['google_distance_notoll'] = g_no_toll_route.distance
d['ratio_notoll'] = float(
tn_no_toll_route.distance) / g_no_toll_route.distance
d['ratio_telenav'] = float(
tn_no_toll_route.distance) / tn_toll_route.distance
d['ratio_google'] = float(
g_no_toll_route.distance) / g_toll_route.distance
d['ratio_t2g'] = d['ratio_telenav'] / d['ratio_google']
print 'avoid toll compare result:', jsonutils.dumps(d)
return d
def sendjson(self, method, urlpath, obj):
headers = {'Content-Type': 'application/json'}
url = '/'.join([self.url, urlpath])
log_msg = "Sending METHOD (%(method)s) URL (%(url)s) JSON (%(obj)s)"%({'method': method, 'url': url, 'obj': obj})
LOG.info(log_msg)
if method=="get":
r = requests.get(url,headers=headers, auth=self.auth)
elif method == "post":
data = jsonutils.dumps(obj, indent=2) if obj else None
r = requests.post(url,data,headers=headers, auth=self.auth)
if r.status_code >= 200 or r.status_code < 299:
return r
else:
r.raise_for_status()
def sendjson(self, method, urlpath, obj):
headers = {'Content-Type': 'application/json'}
url = '/'.join([self.url, urlpath])
log_msg = "Sending METHOD (%(method)s) URL (%(url)s) JSON (%(obj)s)" % (
{
'method': method,
'url': url,
'obj': obj
})
LOG.info(log_msg)
if method == "get":
r = requests.get(url, headers=headers, auth=self.auth)
elif method == "post":
data = jsonutils.dumps(obj, indent=2) if obj else None
r = requests.post(url, data, headers=headers, auth=self.auth)
if r.status_code >= 200 or r.status_code < 299:
return r
else:
r.raise_for_status()
def getNearbyCities(self, lat, lng, extension=1.0, limit=sys.maxint):
''' get adjacent cities near (lat, lng)
@param lat : latitude of the anchor point
@param lng : longitude of the anchor point
@param extension : constrant of extension
@param limit : constrant of number of candidates in return
@return: cities. [{
'id' : city['id'],
'lat' : city['lat'],
'lon' : city['lon'],
'name' : name,
'population' : getPopulation(city)
}]
'''
key = self.getCacheKey({
'lat': lat,
'lng': lng,
'extension': extension,
'limit': limit
})
city_centers = cacheutils.retrieve(key, 'tmp/place')
if city_centers:
return jsonutils.loads(city_centers)
# left top part
(ltLat, ltLon) = self.normalizeLatLon(lat - extension, lng - extension)
ltCities = self.getCitiesInBoundary(ltLat,
ltLon,
lat,
lng,
limit,
anchor={
'lat': lat,
'lng': lng
})
tmpExtension = extension
while len(ltCities) == 0:
print 'expand extension on left top'
tmpExtension *= 2
(ltLat1, ltLon1) = self.normalizeLatLon(lat - tmpExtension,
lng - tmpExtension)
if ltLat1 == ltLat and ltLon1 == ltLon:
print 'WARNING: cannot find nearby [%s] cities on left top part' % latlon
break
ltLat = ltLat1
ltLon = ltLon1
ltCities = self.getCitiesInBoundary(ltLat,
ltLon,
lat,
lng,
limit,
anchor={
'lat': lat,
'lng': lng
})
# rigth top part
(rtLat, rtLon) = self.normalizeLatLon(lat - extension, lng + extension)
rtCities = self.getCitiesInBoundary(rtLat,
lng,
lat,
rtLon,
limit,
anchor={
'lat': lat,
'lng': lng
})
tmpExtension = extension
while len(rtCities) == 0:
print 'expand extension on right top'
tmpExtension *= 2
(rtLat1, rtLon1) = self.normalizeLatLon(lat - tmpExtension,
lng + tmpExtension)
if rtLat1 == rtLat and rtLon1 == rtLon:
print 'WARNING: cannot find nearby [%s] cities on right top part' % latlon
break
rtLat = rtLat1
rtLon = rtLon1
rtCities = self.getCitiesInBoundary(rtLat,
lng,
lat,
rtLon,
limit,
anchor={
'lat': lat,
'lng': lng
})
# left bottom part
(lbLat, lbLon) = self.normalizeLatLon(lat + extension, lng - extension)
lbCities = self.getCitiesInBoundary(lat,
lbLon,
lbLat,
lng,
limit,
anchor={
'lat': lat,
'lng': lng
})
tmpExtension = extension
while len(lbCities) == 0:
print 'expand extension on left bottom'
tmpExtension *= 2
(lbLat1, lbLon1) = self.normalizeLatLon(lat + tmpExtension,
lng - tmpExtension)
if lbLat1 == lbLat and lbLon1 == lbLon:
print 'WARNING: cannot find nearby [%s] cities on left bottom part' % latlon
break
lbLat = lbLat1
lbLon = lbLon1
lbCities = self.getCitiesInBoundary(lat,
lbLon,
lbLat,
lng,
limit,
anchor={
'lat': lat,
'lng': lng
})
# right bottom
(rbLat, rbLon) = self.normalizeLatLon(lat + extension, lng + extension)
rbCities = self.getCitiesInBoundary(lat,
lng,
rbLat,
rbLon,
limit,
anchor={
'lat': lat,
'lng': lng
})
tmpExtension = extension
while len(rbCities) == 0:
print 'expand extension on right bottom'
tmpExtension *= 2
(rbLat1, rbLon1) = self.normalizeLatLon(lat + tmpExtension,
lng + tmpExtension)
if rbLat1 == rbLat and rbLon1 == rbLon:
print 'WARNING: cannot find nearby [%s] cities on right bottom part' % latlon
break
rbLat = rbLat1
rbLon = rbLon1
rbCities = self.getCitiesInBoundary(lat,
lng,
rbLat,
rbLon,
limit,
anchor={
'lat': lat,
'lng': lng
})
city_centers = []
# add at least one candidate in each region
print 'left top cities: ', ltCities
print 'right top cities: ', rtCities
print 'left bottom cities: ', lbCities
print 'right bottom cities: ', rbCities
self.moveTop(city_centers, ltCities)
self.moveTop(city_centers, rtCities)
self.moveTop(city_centers, lbCities)
self.moveTop(city_centers, rbCities)
# print '4 cities :', city_centers
while len(city_centers) < limit:
nextCity = None
if len(ltCities) > 0:
nextCity = ltCities[0]
if len(rtCities) > 0:
if nextCity is None:
nextCity = rtCities[0]
else:
if nextCity['population'] < rtCities[0]['population']:
nextCity = rtCities[0]
if len(lbCities) > 0:
if nextCity is None:
nextCity = lbCities[0]
else:
if nextCity['population'] < lbCities[0]['population']:
nextCity = lbCities[0]
if len(rbCities) > 0:
if nextCity is None:
nextCity = rbCities[0]
else:
if nextCity['population'] < rbCities[0]['population']:
nextCity = rbCities[0]
if nextCity is None:
break
city_centers.append(nextCity)
if nextCity in ltCities:
ltCities.remove(nextCity)
if nextCity in rtCities:
rtCities.remove(nextCity)
if nextCity in lbCities:
lbCities.remove(nextCity)
if nextCity in rbCities:
rbCities.remove(nextCity)
print '\nfind city centers:', city_centers
#dumpAdjacentCitiesKML(wayid, latlon, city_centers)
cacheutils.store(key, 'tmp/place', jsonutils.dumps(city_centers))
return city_centers
def to_json(self, data):
return jsonutils.dumps(data, default=self._sanitizer)
def getODPairsForTollCase(self,
cities_start,
cities_end,
tollcase,
limit=10,
useCache=True):
''' get origin->destination pairs passing toll ways in tollcase['ways']
@param cities_start: cities list near start point
@param cities_end: cities list near start point
@param tollcase: {
'start_point' : {
'lat' : lat,
'lng' : lng
},
'end_point' : {
'lat' : lat,
'lng' : lng
},
'ids' : [
wayid0, wayid1, ...
]
}
@param limit: constraint of orig->dest pair count in return
@param useCache: if True, try to get result from cache
cities is list of {
'id' : id,
'lat' : lat,
'lon' : lon,
'name' : name,
'population' : population
}
@return: { 'orig2dests': pairs }
orig2dests is list of orig->dest pair. [{
'orig' : {
'name':city['name'],
'lat':city['lat'],
'lon':city['lon']
},
'dest' : {
'name':c2['name'],
'lat':c2['lat'],
'lon':c2['lon']
}
}]
'''
# print 'tollcase:', tollcase
key = self.getTollCaseCacheKey(tollcase)
orig2dests = cacheutils.retrieve(key, 'cases', 'tollcase')
print 'len(orig2dests):', 0 if None == orig2dests else len(orig2dests)
if orig2dests:
cases_good = jsonutils.loads(orig2dests)
if len(cases_good) > 0:
return cases_good
elif g_skip_empty_case:
# print 'g_skip_empty_case is True, return'
return cases_good
ms = mapservice.MapService()
cases_good = []
cases_cannot_avoid_toll = []
cases_do_not_through_toll = []
cases_failed = []
# set for wayids in this test case
cur_tollset = set(tollcase['ids'])
#print
#print 'tollset:', cur_tollset
#print 'len(s):', len(cur_tollset), '\tlen(l):', len(tollcase['ids'])
# all toll wayids in final origin->destination pairs
passingset = set()
# all common toll wayids in final origin->destination and cur_tollset
commonset = set()
for city in cities_start:
# print '\tname', city['name']
# print '\tlat,lon = {%f, %f}' % (city['lat'], city['lon'])
for c2 in cities_end:
if len(cases_good) >= limit:
break
if city != c2:
print '\t', city['name'], '->', c2['name']
print '\t{%f,%f}->{%f,%f}' % (city['lat'], city['lon'],
c2['lat'], c2['lon'])
orig = LatLon(city['lat'], city['lon'])
dest = LatLon(c2['lat'], c2['lon'])
output = ms.directions(orig, dest)
# print 'output:', output
pair = {
'orig': {
'name': city['name'],
'lat': city['lat'],
'lon': city['lon']
},
'dest': {
'name': c2['name'],
'lat': c2['lat'],
'lon': c2['lon']
}
}
if 'java.net.SocketTimeoutException: Read timed out' in output:
print '\tFAILED: {%f,%f}->{%f,%f}' % (
city['lat'], city['lon'], c2['lat'], c2['lon'])
cases_failed.append(
'\tFAILED: {%f,%f}->{%f,%f}' %
(city['lat'], city['lon'], c2['lat'], c2['lon']))
continue
# passing_wayids = maputils.edgesInRouteResponse(tollcase['ids'], output)
passing_wayids = maputils.edgesInRouteResponse(
g_tollset, output)
tmpset = set(passing_wayids)
tmpcommonset = tmpset & cur_tollset
if 0 < len(tmpcommonset):
print 'passing wayids: ', passing_wayids
commonset |= tmpcommonset
if 0 == len(tmpset - cur_tollset):
# TODO, zexings, modify edges in route response to collect all the toll edges
print 'all toll ways are in testcase'
else:
print 'WARNING. not all toll ways in tollcase'
print 'common:', tmpcommonset
print 'uncommon:', tmpset - cur_tollset
# exit(0)
print '\tOK: {%f,%f}->{%f,%f}' % (
city['lat'], city['lon'], c2['lat'], c2['lon'])
is_new_case = True
for goodcase in cases_good[:]:
tmp2set = set(goodcase['ids'])
# print 'len(tmp2set):', len(tmp2set)
if len(tmpset - tmp2set) == 0:
# all toll edge can be find in previous origin -> destination case, skip it
# print 'all toll edge can be find in previous origin -> destination case, skip it'
is_new_case = False
break
elif len(tmp2set - tmpset) == 0:
# all toll edge can be find in current origin -> destination case, remove old one
# print 'all toll edge can be find in current origin -> destination case, remove old one'
cases_good.remove(goodcase)
# print 'len(tmpset):', len(tmpset)
# print 'is_new_case:', is_new_case
# print 'len(cases_good):', len(cases_good)
# exit(0)
if not is_new_case:
continue
pair['ids'] = passing_wayids
passingset |= tmpset
cases_good.append(pair)
# exit(0)
# TODO, zexings
# output2 = ms.directions(orig, dest, True)
# if 'java.net.SocketTimeoutException: Read timed out' in output2:
# print '\tFAILED: {%f,%f}-notoll->{%f,%f}' % (city['lat'], city['lon'], c2['lat'], c2['lon'])
# cases_failed.append('\tFAILED: {%f,%f}<-{%f,%f}' % (city['lat'], city['lon'], c2['lat'], c2['lon']))
# continue
#
# if not maputils.edgesInRouteResponse(wayid, output2):
# print '\tGOOD: {%f,%f}-notoll->{%f,%f}' % (city['lat'], city['lon'], c2['lat'], c2['lon'])
# # TODO, generate kml for these two paths
# cases_good.append(pair)
# else:
# print '\tNoAvoid: {%f,%f}-notoll->{%f,%f}' % (city['lat'], city['lon'], c2['lat'], c2['lon'])
# cases_cannot_avoid_toll.append(pair)
# # cannot avoid toll edge
# print 'cannot avoid toll edge'
else:
print '\tNoVia: {%f,%f}->{%f,%f}' % (
city['lat'], city['lon'], c2['lat'], c2['lon'])
cases_do_not_through_toll.append(pair)
print 'good case:', cases_good
print 'cannot avoid toll cases:', cases_cannot_avoid_toll
print 'no toll cases:', cases_do_not_through_toll
print 'failed cases:', cases_failed
print 'coverage:', len(commonset), '/', len(cur_tollset), '=', (
len(commonset) * 1.0 / len(cur_tollset))
d = {'tollcase': tollcase, 'orig2dests': cases_good}
# dump result
# if len(cases_good) > 0:
# cacheutils.store(key, 'cases', jsonutils.dumps(cases_good), 'tollcase')
# store result to avoid duplicate calculation
cacheutils.store(key, 'cases', jsonutils.dumps(cases_good), 'tollcase')
return cases_good
def getTollCaseCacheKey(self, tollcase):
''' get toll case cache key
'''
return jsonutils.dumps(tollcase)
# 2. get all adjacent cities around each toll
# case_cities = getAllAdjacentCities(cases, cfg['extension'], cfg['max'])
# print
#print 'case_cities:\n',case_cities
g_skip_empty_case = True
tollcases = []
g = TollCaseGenerator()
for tc in cases:
odpair = g.generate(tc)
tollcases.append({'toll_info': tc, 'orig2dest': odpair})
# tollcases = getTollCases(cases, cfg, args.placeservice)
print '===================================='
for tc in tollcases:
if len(tc['orig2dest']) == 0:
print jsonutils.dumps(tc)
exit(0)
time.sleep(3)
g_skip_empty_case = False
for tc in tollcases:
if 0 < len(tc['orig2dest']):
continue
# use OverpassPlaceService to find orig->dest test case for this toll
odpair = g.generate(tc['toll_info'],
clz=placeservice.OverpassPlaceService)
print
print 'toll:', tc['toll_info']
print
print 'odpair:', odpair
# exit(0)
exit(0)
n += len(odpairs)
# print 'odpairs:', odpairs
for p in odpairs:
# # dict is not hashable, we use json representation as key
# k = jsonutils.dumps(p)
# # print 'pair:', k
# if k in odpairset:
# dups += 1
# odpairset.add(k)
if p in ods:
dups += 1
continue
ods.append(p)
# exit(0)
# print 'n:', n
# print 'dups:', dups
print 'len(ods):', len(ods)
# exit(0)
return ods
if __name__ == '__main__':
print 'toll case validator'
odpairs = mergeTollCases()
v = TollCaseValidator()
for p in odpairs:
orig = p['orig']
dest = p['dest']
print '%s -> %s' % (jsonutils.dumps(orig), jsonutils.dumps(dest))
v.validate(orig, dest)
def default(self, data):
return jsonutils.dumps(data)