def crawlAreaLocData(): """ 1) fetch 100 records with flag area_ok = 0. 2) try areaLocation(laccid), if OK, then update flag area_ok =1 and quit; else goto 2). 3) try googleAreaLocation(latlon), if OK, then get geoaddr:[province,city,district]; else |wpp_uprecsinfo|.area_try += 1 and quit. 4) search area_code for the found district, insert area location (laccid,areacode,areaname_cn) into |wpp_cellarea|, and update flag area_ok = 1. """ fail_history = {} dbips = DB_OFFLINE for dbip in dbips: dbsvr = dbsvrs[dbip] wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype']) # select config.CRAWL_LIMIT raw fps which haven't tried for google area location. fps_noarea = wppdb.getCrawlFPs() for fp in fps_noarea: # try areaLocation(laccid) laccid = '%s-%s' % (fp[8], fp[9]) if laccid in fail_history: continue time = fp[2] print laccid, time if wppdb.areaLocation(laccid): # area_ok = 1 & quit. wppdb.setUprecsAreaStatus(status=1, time=time) else: print fp # try google area location. geoaddr = googleAreaLocation(latlon=(fp[11], fp[12])) # area_try += 1 & quit wppdb.setUprecAreaTry(area_try=fp[18] + 1, time=time) if geoaddr: # insert area location info(laccid~geoaddr) into |wpp_cellarea|. # till now, area_location: 'laccid,area_code,province>city>district'. area_location = wppdb.addAreaLocation(laccid=laccid, geoaddr=geoaddr) if not area_location: if not laccid in fail_history: fail_history[laccid] = geoaddr print 'Failed to add area location: [%s] for cell[%s]' % \ (geoaddr[-1].encode('utf8'), laccid) continue # area_ok = 1 & quit. wppdb.setUprecsAreaStatus(status=1, time=time) print area_location.encode( 'utf8') # encode('utf8') for crontab. else: if geoaddr is None: sys.exit(0) # OVER_QUERY_LIMIT. else: pass
def crawlAreaLocData(): """ 1) fetch 100 records with flag area_ok = 0. 2) try areaLocation(laccid), if OK, then update flag area_ok =1 and quit; else goto 2). 3) try googleAreaLocation(latlon), if OK, then get geoaddr:[province,city,district]; else |wpp_uprecsinfo|.area_try += 1 and quit. 4) search area_code for the found district, insert area location (laccid,areacode,areaname_cn) into |wpp_cellarea|, and update flag area_ok = 1. """ fail_history = {} dbips = DB_OFFLINE for dbip in dbips: dbsvr = dbsvrs[dbip] wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype']) # select config.CRAWL_LIMIT raw fps which haven't tried for google area location. fps_noarea = wppdb.getCrawlFPs() for fp in fps_noarea: # try areaLocation(laccid) laccid = '%s-%s' % (fp[8], fp[9]) if laccid in fail_history: continue time = fp[2] print laccid, time if wppdb.areaLocation(laccid): # area_ok = 1 & quit. wppdb.setUprecsAreaStatus(status=1, time=time) else: print fp # try google area location. geoaddr = googleAreaLocation( latlon=(fp[11], fp[12]) ) # area_try += 1 & quit wppdb.setUprecAreaTry(area_try=fp[18]+1, time=time) if geoaddr: # insert area location info(laccid~geoaddr) into |wpp_cellarea|. # till now, area_location: 'laccid,area_code,province>city>district'. area_location = wppdb.addAreaLocation(laccid=laccid, geoaddr=geoaddr) if not area_location: if not laccid in fail_history: fail_history[laccid] = geoaddr print 'Failed to add area location: [%s] for cell[%s]' % \ (geoaddr[-1].encode('utf8'), laccid) continue # area_ok = 1 & quit. wppdb.setUprecsAreaStatus(status=1, time=time) print area_location.encode('utf8') # encode('utf8') for crontab. else: if geoaddr is None: sys.exit(0) # OVER_QUERY_LIMIT. else: pass
def fixPos(posreq=None, has_google=False, mc=None): xmlnodes = xmlparser(posreq).getchildren() # Parameters default vals init. lat, lon, ee = 39.9055, 116.3914, 5000 errinfo = 'AccuTooBad'; errcode = '102' # logic control switch init. pos_area = pos_pt = False # Default *PosLevel* is Point if not specified. # WppDB connection init. dbsvr = dbsvrs[DB_ONLINE] wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype']) # lambda func init. f = lambda x : [ node.attrib for node in xmlnodes if node.tag == x ] plevel = f('PosLevel') # Area location related parameters interpretation & default vals init. plevel = plevel[0]['val'] if plevel else 'Point' acode = addr = '' if plevel == 'Hybrid': pos_area = pos_pt = True elif plevel == 'Area': pos_area = True else: pos_pt = True plevel = 'Point' # PosLevel default *Point*. if pos_area: # Area location. cell = f('CellInfo') if cell: laccid = '%s-%s' % (cell[0]['lac'], cell[0]['cid']) acode_addr = wppdb.areaLocation(laccid) if acode_addr: acode, addr = acode_addr errinfo='OK'; errcode='100' lat = lon = ee = '' if pos_pt: # Point location, which returns 3d coordinates. macs = f('WLANIdentifier'); rsss = f('WLANMatcher'); need_google = False; if macs and rsss: macs = macs[0]['val'].split('|') rsss = rsss[0]['val'].split('|') INTERSET = min(CLUSTERKEYSIZE, len(macs)) idxs_max = argsort(rsss)[:INTERSET] macsrsss = vstack((macs, rsss))[:,idxs_max] wlanloc = fixPosWLAN(INTERSET, macsrsss, wppdb, DEBUG_ALGO) if not wlanloc: need_google = True else: wlanloc = [] if not wlanloc: if not pos_area: cell = f('CellInfo') if cell: if not pos_area: laccid = '%s-%s' % (cell[0]['lac'], cell[0]['cid']) celloc = wppdb.laccidLocation(laccid) if not celloc: need_google = True wpplog.error('Cell location FAILED!') elif celloc[2] > GOOG_ERR_LIMIT: need_google = False # googleLocation err too big for wlanloc. else: pass else: celloc = [] loc = wlanloc or celloc if loc: lat, lon, ee = loc errinfo = 'OK'; errcode = '100' # TODO: make googleLocation async job when wlanloc fails & celloc succeeds. # Try Google location, when wifi location failed && wifi info exists. if need_google and has_google: loc_google = googleLocation(macs=macs, rsss=rsss, cellinfo=cell[0], mc=mc) if loc_google: lat1, lon1, h, ee_goog = loc_google if not loc: lat, lon, ee = lat1, lon1, ee_goog errinfo = 'OK'; errcode = '100' # wifi location import. TODO: make google loc import job async when it's *succeeded*. if macs and ee_goog <= GOOG_ERR_LIMIT: t = f('Time') t = t[0]['val'] if t else '' fp = '1000, 1000101, %s%s%s, %s, %s, %s, %s' % \ (t,','*9,lat1, lon1, h, '|'.join(macs), '|'.join(rsss)) n = doClusterIncr(fd_csv=StringIO(fp), wppdb=wppdb, verb=False) if n['n_newfps'] == 1: wpplog.info('Added 1 WLAN FP from Google') else: wpplog.error('Failed to add FP from Google!') # Cell location import. if cell and not celloc: if ee_goog <= GOOG_ERR_LIMIT: loc_google[-1] = 500 wppdb.addCellLocation(laccid=laccid, loc=loc_google) wpplog.info('Added 1 Cell FP from Google') else: wpplog.error('Google location FAILED!') wppdb.close() if plevel == 'Hybrid': posresp = POS_RESP_FULL % (errcode, errinfo, lat, lon, ee, plevel, acode, addr) elif plevel == 'Area': posresp = POS_RESP_AREA % (errcode, errinfo, plevel, acode, addr) else: posresp = POS_RESP_PT % (errcode, errinfo, lat, lon, ee, plevel) return posresp