def other2baidu(lng, lat, maptype): if maptype == u'soso': lng, lat = _coordtrans.coordtrans('gcj02', 'bd09ll', float(lng), float(lat)) elif maptype == u'wgs': lng, lat = _coordtrans.coordtrans('wgs84', 'bd09ll', float(lng), float(lat)) else: lng, lat = float(lng), float(lat) return lng, lat
def parseHadoopFile(filename, r): print "fileName:", filename open_file = open(filename, "r") i = 0 num = 0 setnum = 0 strCmd = {} # pipe = r.pipeline(transaction=False) for line in open_file: i += 1 strline = re.sub(" {1,}", " ", line) vec = re.split(" |#|\t", strline) if 5 != len(vec): continue num += 1 setnum += 1 # transfer to soso coord y, x = string.atof(vec[2]), string.atof(vec[1]) (x, y) = coordtrans("bd09ll", "gcj02", x, y) y1, x1 = string.atof(vec[4]), string.atof(vec[3]) (x1, y1) = coordtrans("bd09ll", "gcj02", x1, y1) # print "baidu: ",vec[1],vec[2],vec[3],vec[4] # print "soso: ",x,y,x1,y1 userAddr = userInfo_pb2.UserAddr() userAddr.userid = string.atol(vec[0]) userAddr.homelng = x userAddr.homelat = y userAddr.corplng = x1 userAddr.corplat = y1 key = "addr" + vec[0] value = userAddr.SerializeToString() # r.set(key, value) # print key,value strCmd[key] = value if setnum >= 200: # print "setnum=",setnum,strCmd r.mset(strCmd) strCmd.clear() setnum = 0 if strCmd: # print "last setnum=%d"%(setnum) r.mset(strCmd) strCmd.clear() setnum = 0 print "hadoopFile,", filename, "lines=", i, "num=", num open_file.close()
def parseMultiDestFile(filename, r): print "fileName:",filename open_file = open(filename, 'r') i = 0 num = 0 count = 0 didiDestAddr1 = userInfo_pb2.DidiDestAddr() didiDestAddr1.Clear() for line in open_file: i += 1 vec = line.split(',') if (len(vec) < 7): #print "--------------------------less than 7",len(vec) continue first = int(vec[0]) if (0 != first): num += 1 #print vec[0],vec[4],vec[5] id = string.atol(vec[0]) #coordtrans: from baidu to soso y,x = string.atof(vec[5]), string.atof(vec[4]) (x, y) = coordtrans('bd09ll', 'gcj02', x, y) #print "befor:x=%lf,y=%lf"%(string.atof(vec[4]), string.atof(vec[5])) #print "after:x=%lf,y=%lf"%(x, y) if (1 == num): didiDestAddr1.userid = id destAddr = didiDestAddr1.adestaddr.add() destAddr.tolng = x #string.atof(vec[4]) destAddr.tolat = y #string.atof(vec[5]) else: if (didiDestAddr1.userid == id): destAddr = didiDestAddr1.adestaddr.add() destAddr.tolng = x #string.atof(vec[4]) destAddr.tolat = y #string.atof(vec[5]) #print "repeate id",id else: count += 1 key = "dest" + str(didiDestAddr1.userid) value = didiDestAddr1.SerializeToString() r.set(key, value) print "set id",didiDestAddr1.userid #clear the old and add a new element didiDestAddr1.Clear() didiDestAddr1.userid = id destAddr = didiDestAddr1.adestaddr.add() destAddr.tolng = x #string.atof(vec[4]) destAddr.tolat = y #string.atof(vec[5]) #the last line if (0 != didiDestAddr1.userid): key = "dest" + str(didiDestAddr1.userid) value = didiDestAddr1.SerializeToString() r.set(key, value) print "set id",didiDestAddr1.userid print "DestAddrFile lines=",i," num=",num,"count=",count open_file.close()
line = sys.stdin.readline(); if line == "" : break; i += 1 vec = line.split(',') if (len(vec) != 7): #print "--------------------------less than 7",len(vec) continue first = int(vec[0]) if (0 != first): num += 1 id = string.atol(vec[0]) #coordtrans: from baidu to soso y,x = string.atof(vec[5]), string.atof(vec[4]) (x, y) = coordtrans('bd09ll', 'gcj02', x, y) #print "befor:x=%lf,y=%lf"%(string.atof(vec[4]), string.atof(vec[5])) #print "after:x=%lf,y=%lf"%(x, y) if (1 == num): didiDestAddr1.userid = id destAddr = didiDestAddr1.adestaddr.add() destAddr.tolng = x #string.atof(vec[4]) destAddr.tolat = y #string.atof(vec[5]) else: if (didiDestAddr1.userid == id): destAddr = didiDestAddr1.adestaddr.add() destAddr.tolng = x #string.atof(vec[4]) destAddr.tolat = y #string.atof(vec[5]) else: count += 1
def parseMultiStartFile(filename, r): start = datetime.datetime.now() print "starttime=%s",start print "fileName:",filename open_file = open(filename, 'r') i = 0 num = 0 count = 0 didiStartAddr1 = userInfo_pb2.DidiStartAddr() didiStartAddr1.Clear() for line in open_file: #for line in open_file.readlines(): i += 1 vec = line.split(',') if (7 != len(vec)): #print "-----less than 7",len(vec) continue first = int(vec[0]) if (0 != first): num += 1 #print vec[0],vec[4],vec[5] id = string.atol(vec[0]) #coordtrans: from baidu to soso y,x = string.atof(vec[5]), string.atof(vec[4]) (x, y) = coordtrans('bd09ll', 'gcj02', x, y) if (1 == num): didiStartAddr1.userid = id startAddr = didiStartAddr1.astartaddr.add() startAddr.fromlng = x #string.atof(vec[4]) startAddr.fromlat = y #string.atof(vec[5]) else: if (didiStartAddr1.userid == id): startAddr = didiStartAddr1.astartaddr.add() startAddr.fromlng = x #string.atof(vec[4]) startAddr.fromlat = y #string.atof(vec[5]) #print "repeate id",id else: count += 1 key = "start" + str(didiStartAddr1.userid) value = didiStartAddr1.SerializeToString() r.set(key, value) print "set id",didiStartAddr1.userid #clear the old and add a new element didiStartAddr1.Clear() didiStartAddr1.userid = id startAddr = didiStartAddr1.astartaddr.add() startAddr.fromlng = x #string.atof(vec[4]) startAddr.fromlat = y #string.atof(vec[5]) #the last line if (0 != didiStartAddr1.userid): key = "start" + str(didiStartAddr1.userid) value = didiStartAddr1.SerializeToString() r.set(key, value) print "set id",didiStartAddr1.userid print "StartAddrFile lines=",i," num=",num,"count=",count open_file.close() end = datetime.datetime.now() print "endtime=%s",end print "start Time consume:",(end - start).seconds