def crawlAreaLocData():
"""
1) fetch 100 records with flag area_ok = 0.
2) try areaLocation(laccid), if OK, then update flag area_ok =1 and quit; else goto 2).
3) try googleAreaLocation(latlon), if OK, then get geoaddr:[province,city,district];
else |wpp_uprecsinfo|.area_try += 1 and quit.
4) search area_code for the found district, insert area location
(laccid,areacode,areaname_cn) into |wpp_cellarea|, and update flag area_ok = 1.
"""
fail_history = {}
dbips = DB_OFFLINE
for dbip in dbips:
dbsvr = dbsvrs[dbip]
wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype'])
# select config.CRAWL_LIMIT raw fps which haven't tried for google area location.
fps_noarea = wppdb.getCrawlFPs()
for fp in fps_noarea:
# try areaLocation(laccid)
laccid = '%s-%s' % (fp[8], fp[9])
if laccid in fail_history: continue
time = fp[2]
print laccid, time
if wppdb.areaLocation(laccid):
# area_ok = 1 & quit.
wppdb.setUprecsAreaStatus(status=1, time=time)
else:
print fp
# try google area location.
geoaddr = googleAreaLocation(latlon=(fp[11], fp[12]))
# area_try += 1 & quit
wppdb.setUprecAreaTry(area_try=fp[18] + 1, time=time)
if geoaddr:
# insert area location info(laccid~geoaddr) into |wpp_cellarea|.
# till now, area_location: 'laccid,area_code,province>city>district'.
area_location = wppdb.addAreaLocation(laccid=laccid,
geoaddr=geoaddr)
if not area_location:
if not laccid in fail_history:
fail_history[laccid] = geoaddr
print 'Failed to add area location: [%s] for cell[%s]' % \
(geoaddr[-1].encode('utf8'), laccid)
continue
# area_ok = 1 & quit.
wppdb.setUprecsAreaStatus(status=1, time=time)
print area_location.encode(
'utf8') # encode('utf8') for crontab.
else:
if geoaddr is None: sys.exit(0) # OVER_QUERY_LIMIT.
else: pass
def crawlAreaLocData():
"""
1) fetch 100 records with flag area_ok = 0.
2) try areaLocation(laccid), if OK, then update flag area_ok =1 and quit; else goto 2).
3) try googleAreaLocation(latlon), if OK, then get geoaddr:[province,city,district];
else |wpp_uprecsinfo|.area_try += 1 and quit.
4) search area_code for the found district, insert area location
(laccid,areacode,areaname_cn) into |wpp_cellarea|, and update flag area_ok = 1.
"""
fail_history = {}
dbips = DB_OFFLINE
for dbip in dbips:
dbsvr = dbsvrs[dbip]
wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype'])
# select config.CRAWL_LIMIT raw fps which haven't tried for google area location.
fps_noarea = wppdb.getCrawlFPs()
for fp in fps_noarea:
# try areaLocation(laccid)
laccid = '%s-%s' % (fp[8], fp[9])
if laccid in fail_history: continue
time = fp[2]
print laccid, time
if wppdb.areaLocation(laccid):
# area_ok = 1 & quit.
wppdb.setUprecsAreaStatus(status=1, time=time)
else:
print fp
# try google area location.
geoaddr = googleAreaLocation( latlon=(fp[11], fp[12]) )
# area_try += 1 & quit
wppdb.setUprecAreaTry(area_try=fp[18]+1, time=time)
if geoaddr:
# insert area location info(laccid~geoaddr) into |wpp_cellarea|.
# till now, area_location: 'laccid,area_code,province>city>district'.
area_location = wppdb.addAreaLocation(laccid=laccid, geoaddr=geoaddr)
if not area_location:
if not laccid in fail_history:
fail_history[laccid] = geoaddr
print 'Failed to add area location: [%s] for cell[%s]' % \
(geoaddr[-1].encode('utf8'), laccid)
continue
# area_ok = 1 & quit.
wppdb.setUprecsAreaStatus(status=1, time=time)
print area_location.encode('utf8') # encode('utf8') for crontab.
else:
if geoaddr is None: sys.exit(0) # OVER_QUERY_LIMIT.
else: pass
def fixPos(posreq=None, has_google=False, mc=None):
xmlnodes = xmlparser(posreq).getchildren()
# Parameters default vals init.
lat, lon, ee = 39.9055, 116.3914, 5000
errinfo = 'AccuTooBad'; errcode = '102'
# logic control switch init.
pos_area = pos_pt = False # Default *PosLevel* is Point if not specified.
# WppDB connection init.
dbsvr = dbsvrs[DB_ONLINE]
wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype'])
# lambda func init.
f = lambda x : [ node.attrib for node in xmlnodes if node.tag == x ]
plevel = f('PosLevel')
# Area location related parameters interpretation & default vals init.
plevel = plevel[0]['val'] if plevel else 'Point'
acode = addr = ''
if plevel == 'Hybrid': pos_area = pos_pt = True
elif plevel == 'Area': pos_area = True
else:
pos_pt = True
plevel = 'Point' # PosLevel default *Point*.
if pos_area: # Area location.
cell = f('CellInfo')
if cell:
laccid = '%s-%s' % (cell[0]['lac'], cell[0]['cid'])
acode_addr = wppdb.areaLocation(laccid)
if acode_addr:
acode, addr = acode_addr
errinfo='OK'; errcode='100'
lat = lon = ee = ''
if pos_pt: # Point location, which returns 3d coordinates.
macs = f('WLANIdentifier'); rsss = f('WLANMatcher'); need_google = False;
if macs and rsss:
macs = macs[0]['val'].split('|')
rsss = rsss[0]['val'].split('|')
INTERSET = min(CLUSTERKEYSIZE, len(macs))
idxs_max = argsort(rsss)[:INTERSET]
macsrsss = vstack((macs, rsss))[:,idxs_max]
wlanloc = fixPosWLAN(INTERSET, macsrsss, wppdb, DEBUG_ALGO)
if not wlanloc:
need_google = True
else: wlanloc = []
if not wlanloc:
if not pos_area: cell = f('CellInfo')
if cell:
if not pos_area:
laccid = '%s-%s' % (cell[0]['lac'], cell[0]['cid'])
celloc = wppdb.laccidLocation(laccid)
if not celloc:
need_google = True
wpplog.error('Cell location FAILED!')
elif celloc[2] > GOOG_ERR_LIMIT:
need_google = False # googleLocation err too big for wlanloc.
else: pass
else: celloc = []
loc = wlanloc or celloc
if loc:
lat, lon, ee = loc
errinfo = 'OK'; errcode = '100'
# TODO: make googleLocation async job when wlanloc fails & celloc succeeds.
# Try Google location, when wifi location failed && wifi info exists.
if need_google and has_google:
loc_google = googleLocation(macs=macs, rsss=rsss, cellinfo=cell[0], mc=mc)
if loc_google:
lat1, lon1, h, ee_goog = loc_google
if not loc:
lat, lon, ee = lat1, lon1, ee_goog
errinfo = 'OK'; errcode = '100'
# wifi location import. TODO: make google loc import job async when it's *succeeded*.
if macs and ee_goog <= GOOG_ERR_LIMIT:
t = f('Time')
t = t[0]['val'] if t else ''
fp = '1000, 1000101, %s%s%s, %s, %s, %s, %s' % \
(t,','*9,lat1, lon1, h, '|'.join(macs), '|'.join(rsss))
n = doClusterIncr(fd_csv=StringIO(fp), wppdb=wppdb, verb=False)
if n['n_newfps'] == 1:
wpplog.info('Added 1 WLAN FP from Google')
else:
wpplog.error('Failed to add FP from Google!')
# Cell location import.
if cell and not celloc:
if ee_goog <= GOOG_ERR_LIMIT:
loc_google[-1] = 500
wppdb.addCellLocation(laccid=laccid, loc=loc_google)
wpplog.info('Added 1 Cell FP from Google')
else: wpplog.error('Google location FAILED!')
wppdb.close()
if plevel == 'Hybrid': posresp = POS_RESP_FULL % (errcode, errinfo, lat, lon, ee, plevel, acode, addr)
elif plevel == 'Area': posresp = POS_RESP_AREA % (errcode, errinfo, plevel, acode, addr)
else: posresp = POS_RESP_PT % (errcode, errinfo, lat, lon, ee, plevel)
return posresp
