def loadRawdata(rawfile=None, updbmode=1): """ rawfile: rawdata csv file. updbmode: update db mode: 1-all, 2-incr. Init *algo* tables with rawdata csv(16 columns) -- SLOW if csv is big, try offline.doClusterAll(rawdata) -> db.loadClusteredData() instead. 1) db.initTables(): init db tables if update all the db data. 2) db.updateIndexes(): update tables indexes, drop old idxs if only update db incrementally. 3) offline.doClusterIncr(): incremental clustering. """ dbips = DB_OFFLINE doflush = True for dbip in dbips: dbsvr = dbsvrs[dbip] wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype']) if updbmode == 1: # Create WPP tables. wppdb.initTables(doDrop=True) doflush = False # Update indexs. wppdb.updateIndexes(doflush) # Load csv clustered data into DB tables. n_inserts = doClusterIncr(fd_csv=file(rawfile), wppdb=wppdb) print 'Added: [%s] clusters, [%s] FPs' % (n_inserts['n_newcids'], n_inserts['n_newfps']) # Init ver_uprecs in |wpp_uprecsver| if it's empty. if wppdb.getRawdataVersion() is None: wppdb.setRawdataVersion('0') wppdb.close()
def fixPos(posreq=None, has_google=False, mc=None): xmlnodes = xmlparser(posreq).getchildren() # Parameters default vals init. lat, lon, ee = 39.9055, 116.3914, 5000 errinfo = 'AccuTooBad'; errcode = '102' # logic control switch init. pos_area = pos_pt = False # Default *PosLevel* is Point if not specified. # WppDB connection init. dbsvr = dbsvrs[DB_ONLINE] wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype']) # lambda func init. f = lambda x : [ node.attrib for node in xmlnodes if node.tag == x ] plevel = f('PosLevel') # Area location related parameters interpretation & default vals init. plevel = plevel[0]['val'] if plevel else 'Point' acode = addr = '' if plevel == 'Hybrid': pos_area = pos_pt = True elif plevel == 'Area': pos_area = True else: pos_pt = True plevel = 'Point' # PosLevel default *Point*. if pos_area: # Area location. cell = f('CellInfo') if cell: laccid = '%s-%s' % (cell[0]['lac'], cell[0]['cid']) acode_addr = wppdb.areaLocation(laccid) if acode_addr: acode, addr = acode_addr errinfo='OK'; errcode='100' lat = lon = ee = '' if pos_pt: # Point location, which returns 3d coordinates. macs = f('WLANIdentifier'); rsss = f('WLANMatcher'); need_google = False; if macs and rsss: macs = macs[0]['val'].split('|') rsss = rsss[0]['val'].split('|') INTERSET = min(CLUSTERKEYSIZE, len(macs)) idxs_max = argsort(rsss)[:INTERSET] macsrsss = vstack((macs, rsss))[:,idxs_max] wlanloc = fixPosWLAN(INTERSET, macsrsss, wppdb, DEBUG_ALGO) if not wlanloc: need_google = True else: wlanloc = [] if not wlanloc: if not pos_area: cell = f('CellInfo') if cell: if not pos_area: laccid = '%s-%s' % (cell[0]['lac'], cell[0]['cid']) celloc = wppdb.laccidLocation(laccid) if not celloc: need_google = True wpplog.error('Cell location FAILED!') elif celloc[2] > GOOG_ERR_LIMIT: need_google = False # googleLocation err too big for wlanloc. else: pass else: celloc = [] loc = wlanloc or celloc if loc: lat, lon, ee = loc errinfo = 'OK'; errcode = '100' # TODO: make googleLocation async job when wlanloc fails & celloc succeeds. # Try Google location, when wifi location failed && wifi info exists. if need_google and has_google: loc_google = googleLocation(macs=macs, rsss=rsss, cellinfo=cell[0], mc=mc) if loc_google: lat1, lon1, h, ee_goog = loc_google if not loc: lat, lon, ee = lat1, lon1, ee_goog errinfo = 'OK'; errcode = '100' # wifi location import. TODO: make google loc import job async when it's *succeeded*. if macs and ee_goog <= GOOG_ERR_LIMIT: t = f('Time') t = t[0]['val'] if t else '' fp = '1000, 1000101, %s%s%s, %s, %s, %s, %s' % \ (t,','*9,lat1, lon1, h, '|'.join(macs), '|'.join(rsss)) n = doClusterIncr(fd_csv=StringIO(fp), wppdb=wppdb, verb=False) if n['n_newfps'] == 1: wpplog.info('Added 1 WLAN FP from Google') else: wpplog.error('Failed to add FP from Google!') # Cell location import. if cell and not celloc: if ee_goog <= GOOG_ERR_LIMIT: loc_google[-1] = 500 wppdb.addCellLocation(laccid=laccid, loc=loc_google) wpplog.info('Added 1 Cell FP from Google') else: wpplog.error('Google location FAILED!') wppdb.close() if plevel == 'Hybrid': posresp = POS_RESP_FULL % (errcode, errinfo, lat, lon, ee, plevel, acode, addr) elif plevel == 'Area': posresp = POS_RESP_AREA % (errcode, errinfo, plevel, acode, addr) else: posresp = POS_RESP_PT % (errcode, errinfo, lat, lon, ee, plevel) return posresp
def main(): import getopt try: opts, args = getopt.getopt(sys.argv[1:], "ac:f:hi:k:m:nr:st:uv", ["areacrawl","cluster","floor=","help","spid=","kml=","mode=","no-dump", "rawdata","scan","to-rmp=","updatedb","verbose"]) except getopt.GetoptError: usage() sys.exit(99) if not opts: usage(); sys.exit(0) # global vars init. crawl_area=False; updatedb=False; doLoadRawdata=False; scan=False #spid=0; tormp=False; tfail=0; dokml=False; rawfile=None; docluster=False; updbmode=1 global verbose,pp,floor,nodump verbose=False; pp=None; nodump=False; floor=False for o,a in opts: if o in ("-a", "--areacrawl"): crawl_area = True elif o in ("-c", "--cluster"): if not a.isdigit(): print '\ncluster type: %s should be an INTEGER!' % str(a) usage(); sys.exit(99) else: # 1-All; 2-Incr. cluster_type = int(a) docluster = True rmpfile = sys.argv[3] if not os.path.isfile(rmpfile): print 'Raw data file NOT exist: %s!' % rmpfile sys.exit(99) #elif o in ("-i", "--spid"): # if a.isdigit(): spid = int(a) # else: # print '\nspid: %s should be an INTEGER!' % str(a) # usage(); sys.exit(99) elif o in ("-m", "--mode"): if a.isdigit(): updbmode = int(a) if not (1 <= updbmode <= 2): print '\nError: updatedb mode: (%d) NOT supported yet!' % updbmode usage(); sys.exit(99) else: print '\nmode: %s should be an INTEGER!' % str(a) usage(); sys.exit(99) elif o in ("-r", "--rawdata"): if not os.path.isfile(a): print 'Rawdata file NOT exist: %s' % a sys.exit(99) else: doLoadRawdata = True rawfile = a elif o in ("-s", "--scan"): scan = True #elif o in ("-t", "--to-rmp"): # if not os.path.isfile(a): # print 'Raw data file NOT exist: %s' % a # sys.exit(99) # else: # tormp = True # rawfile = a #elif o in ("-k", "--kml"): # if not os.path.isfile(a): # print 'cfprints table file NOT exist: %s' % a # sys.exit(99) # else: # dokml = True # cfpsfile = a #elif o in ("-n", "--no-dump"): # nodump = True elif o in ("-f", "--floor"): if a.isdigit(): floor = int(a) else: print '\nfloor: %s should be an INTEGER!\n' % str(a) usage(); sys.exit(99) elif o in ("-u", "--updatedb"): updatedb = True elif o in ("-v", "--verbose"): verbose = True pp = PrettyPrinter(indent=2) elif o in ("-h", "--help"): usage(); sys.exit(0) else: print 'Parameter NOT supported: %s' % o usage(); sys.exit(99) if doLoadRawdata: loadRawdata(rawfile, updbmode) # Update Algorithm related data. if updatedb: updateAlgoData() if crawl_area: crawlAreaLocData() # Ordinary fingerprints clustering. if docluster: if cluster_type == 1: doClusterAll(file(rmpfile)) elif cluster_type == 2: dbips = DB_OFFLINE for dbip in dbips: dbsvr = dbsvrs[dbip] wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype']) n_inserts = doClusterIncr(fd_csv=file(rmpfile), wppdb=wppdb) print 'Added: [%s] clusters, [%s] FPs' % (n_inserts['n_newcids'], n_inserts['n_newfps']) wppdb.close() else: sys.exit('Unsupported cluster type code: %s!' % cluster_type) # KML generation. #if dokml: # genKMLfile(cfpsfile) ## Raw data to fingerprint convertion. #if tormp: # fingerprint = [] # fingerprint = genFPs(rawfile) # if not fingerprint: # print 'Error: Fingerprint generation FAILED: %s' % rawfile # sys.exit(99) # if nodump is False: # if not rawfile == None: # date = strftime('%Y-%m%d') # rmpfilename = DATPATH + date + RMPSUFFIX # dumpCSV(rmpfilename, fingerprint) # print '-'*65 # sys.exit(0) # else: # usage(); sys.exit(99) # else: # if verbose: pp.pprint(fingerprint) # else: print fingerprint # sys.exit(0) # WLAN scan for FP raw data collection. if scan: collectFPs()
def main(): import getopt try: opts, args = getopt.getopt(sys.argv[1:], "ac:f:hi:k:m:nr:st:uv", [ "areacrawl", "cluster", "floor=", "help", "spid=", "kml=", "mode=", "no-dump", "rawdata", "scan", "to-rmp=", "updatedb", "verbose" ]) except getopt.GetoptError: usage() sys.exit(99) if not opts: usage() sys.exit(0) # global vars init. crawl_area = False updatedb = False doLoadRawdata = False scan = False #spid=0; tormp=False; tfail=0; dokml=False; rawfile = None docluster = False updbmode = 1 global verbose, pp, floor, nodump verbose = False pp = None nodump = False floor = False for o, a in opts: if o in ("-a", "--areacrawl"): crawl_area = True elif o in ("-c", "--cluster"): if not a.isdigit(): print '\ncluster type: %s should be an INTEGER!' % str(a) usage() sys.exit(99) else: # 1-All; 2-Incr. cluster_type = int(a) docluster = True rmpfile = sys.argv[3] if not os.path.isfile(rmpfile): print 'Raw data file NOT exist: %s!' % rmpfile sys.exit(99) #elif o in ("-i", "--spid"): # if a.isdigit(): spid = int(a) # else: # print '\nspid: %s should be an INTEGER!' % str(a) # usage(); sys.exit(99) elif o in ("-m", "--mode"): if a.isdigit(): updbmode = int(a) if not (1 <= updbmode <= 2): print '\nError: updatedb mode: (%d) NOT supported yet!' % updbmode usage() sys.exit(99) else: print '\nmode: %s should be an INTEGER!' % str(a) usage() sys.exit(99) elif o in ("-r", "--rawdata"): if not os.path.isfile(a): print 'Rawdata file NOT exist: %s' % a sys.exit(99) else: doLoadRawdata = True rawfile = a elif o in ("-s", "--scan"): scan = True #elif o in ("-t", "--to-rmp"): # if not os.path.isfile(a): # print 'Raw data file NOT exist: %s' % a # sys.exit(99) # else: # tormp = True # rawfile = a #elif o in ("-k", "--kml"): # if not os.path.isfile(a): # print 'cfprints table file NOT exist: %s' % a # sys.exit(99) # else: # dokml = True # cfpsfile = a #elif o in ("-n", "--no-dump"): # nodump = True elif o in ("-f", "--floor"): if a.isdigit(): floor = int(a) else: print '\nfloor: %s should be an INTEGER!\n' % str(a) usage() sys.exit(99) elif o in ("-u", "--updatedb"): updatedb = True elif o in ("-v", "--verbose"): verbose = True pp = PrettyPrinter(indent=2) elif o in ("-h", "--help"): usage() sys.exit(0) else: print 'Parameter NOT supported: %s' % o usage() sys.exit(99) if doLoadRawdata: loadRawdata(rawfile, updbmode) # Update Algorithm related data. if updatedb: updateAlgoData() if crawl_area: crawlAreaLocData() # Ordinary fingerprints clustering. if docluster: if cluster_type == 1: doClusterAll(file(rmpfile)) elif cluster_type == 2: dbips = DB_OFFLINE for dbip in dbips: dbsvr = dbsvrs[dbip] wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype']) n_inserts = doClusterIncr(fd_csv=file(rmpfile), wppdb=wppdb) print 'Added: [%s] clusters, [%s] FPs' % ( n_inserts['n_newcids'], n_inserts['n_newfps']) wppdb.close() else: sys.exit('Unsupported cluster type code: %s!' % cluster_type) # KML generation. #if dokml: # genKMLfile(cfpsfile) ## Raw data to fingerprint convertion. #if tormp: # fingerprint = [] # fingerprint = genFPs(rawfile) # if not fingerprint: # print 'Error: Fingerprint generation FAILED: %s' % rawfile # sys.exit(99) # if nodump is False: # if not rawfile == None: # date = strftime('%Y-%m%d') # rmpfilename = DATPATH + date + RMPSUFFIX # dumpCSV(rmpfilename, fingerprint) # print '-'*65 # sys.exit(0) # else: # usage(); sys.exit(99) # else: # if verbose: pp.pprint(fingerprint) # else: print fingerprint # sys.exit(0) # WLAN scan for FP raw data collection. if scan: collectFPs()