def other2baidu(lng, lat, maptype):
if maptype == u'soso':
lng, lat = _coordtrans.coordtrans('gcj02', 'bd09ll', float(lng),
float(lat))
elif maptype == u'wgs':
lng, lat = _coordtrans.coordtrans('wgs84', 'bd09ll', float(lng),
float(lat))
else:
lng, lat = float(lng), float(lat)
return lng, lat
def parseHadoopFile(filename, r):
print "fileName:", filename
open_file = open(filename, "r")
i = 0
num = 0
setnum = 0
strCmd = {}
# pipe = r.pipeline(transaction=False)
for line in open_file:
i += 1
strline = re.sub(" {1,}", " ", line)
vec = re.split(" |#|\t", strline)
if 5 != len(vec):
continue
num += 1
setnum += 1
# transfer to soso coord
y, x = string.atof(vec[2]), string.atof(vec[1])
(x, y) = coordtrans("bd09ll", "gcj02", x, y)
y1, x1 = string.atof(vec[4]), string.atof(vec[3])
(x1, y1) = coordtrans("bd09ll", "gcj02", x1, y1)
# print "baidu: ",vec[1],vec[2],vec[3],vec[4]
# print "soso: ",x,y,x1,y1
userAddr = userInfo_pb2.UserAddr()
userAddr.userid = string.atol(vec[0])
userAddr.homelng = x
userAddr.homelat = y
userAddr.corplng = x1
userAddr.corplat = y1
key = "addr" + vec[0]
value = userAddr.SerializeToString()
# r.set(key, value)
# print key,value
strCmd[key] = value
if setnum >= 200:
# print "setnum=",setnum,strCmd
r.mset(strCmd)
strCmd.clear()
setnum = 0
if strCmd:
# print "last setnum=%d"%(setnum)
r.mset(strCmd)
strCmd.clear()
setnum = 0
print "hadoopFile,", filename, "lines=", i, "num=", num
open_file.close()
def parseMultiDestFile(filename, r):
print "fileName:",filename
open_file = open(filename, 'r')
i = 0
num = 0
count = 0
didiDestAddr1 = userInfo_pb2.DidiDestAddr()
didiDestAddr1.Clear()
for line in open_file:
i += 1
vec = line.split(',')
if (len(vec) < 7):
#print "--------------------------less than 7",len(vec)
continue
first = int(vec[0])
if (0 != first):
num += 1
#print vec[0],vec[4],vec[5]
id = string.atol(vec[0])
#coordtrans: from baidu to soso
y,x = string.atof(vec[5]), string.atof(vec[4])
(x, y) = coordtrans('bd09ll', 'gcj02', x, y)
#print "befor:x=%lf,y=%lf"%(string.atof(vec[4]), string.atof(vec[5]))
#print "after:x=%lf,y=%lf"%(x, y)
if (1 == num):
didiDestAddr1.userid = id
destAddr = didiDestAddr1.adestaddr.add()
destAddr.tolng = x #string.atof(vec[4])
destAddr.tolat = y #string.atof(vec[5])
else:
if (didiDestAddr1.userid == id):
destAddr = didiDestAddr1.adestaddr.add()
destAddr.tolng = x #string.atof(vec[4])
destAddr.tolat = y #string.atof(vec[5])
#print "repeate id",id
else:
count += 1
key = "dest" + str(didiDestAddr1.userid)
value = didiDestAddr1.SerializeToString()
r.set(key, value)
print "set id",didiDestAddr1.userid
#clear the old and add a new element
didiDestAddr1.Clear()
didiDestAddr1.userid = id
destAddr = didiDestAddr1.adestaddr.add()
destAddr.tolng = x #string.atof(vec[4])
destAddr.tolat = y #string.atof(vec[5])
#the last line
if (0 != didiDestAddr1.userid):
key = "dest" + str(didiDestAddr1.userid)
value = didiDestAddr1.SerializeToString()
r.set(key, value)
print "set id",didiDestAddr1.userid
print "DestAddrFile lines=",i," num=",num,"count=",count
open_file.close()
line = sys.stdin.readline();
if line == "" :
break;
i += 1
vec = line.split(',')
if (len(vec) != 7):
#print "--------------------------less than 7",len(vec)
continue
first = int(vec[0])
if (0 != first):
num += 1
id = string.atol(vec[0])
#coordtrans: from baidu to soso
y,x = string.atof(vec[5]), string.atof(vec[4])
(x, y) = coordtrans('bd09ll', 'gcj02', x, y)
#print "befor:x=%lf,y=%lf"%(string.atof(vec[4]), string.atof(vec[5]))
#print "after:x=%lf,y=%lf"%(x, y)
if (1 == num):
didiDestAddr1.userid = id
destAddr = didiDestAddr1.adestaddr.add()
destAddr.tolng = x #string.atof(vec[4])
destAddr.tolat = y #string.atof(vec[5])
else:
if (didiDestAddr1.userid == id):
destAddr = didiDestAddr1.adestaddr.add()
destAddr.tolng = x #string.atof(vec[4])
destAddr.tolat = y #string.atof(vec[5])
else:
count += 1
def parseMultiStartFile(filename, r):
start = datetime.datetime.now()
print "starttime=%s",start
print "fileName:",filename
open_file = open(filename, 'r')
i = 0
num = 0
count = 0
didiStartAddr1 = userInfo_pb2.DidiStartAddr()
didiStartAddr1.Clear()
for line in open_file:
#for line in open_file.readlines():
i += 1
vec = line.split(',')
if (7 != len(vec)):
#print "-----less than 7",len(vec)
continue
first = int(vec[0])
if (0 != first):
num += 1
#print vec[0],vec[4],vec[5]
id = string.atol(vec[0])
#coordtrans: from baidu to soso
y,x = string.atof(vec[5]), string.atof(vec[4])
(x, y) = coordtrans('bd09ll', 'gcj02', x, y)
if (1 == num):
didiStartAddr1.userid = id
startAddr = didiStartAddr1.astartaddr.add()
startAddr.fromlng = x #string.atof(vec[4])
startAddr.fromlat = y #string.atof(vec[5])
else:
if (didiStartAddr1.userid == id):
startAddr = didiStartAddr1.astartaddr.add()
startAddr.fromlng = x #string.atof(vec[4])
startAddr.fromlat = y #string.atof(vec[5])
#print "repeate id",id
else:
count += 1
key = "start" + str(didiStartAddr1.userid)
value = didiStartAddr1.SerializeToString()
r.set(key, value)
print "set id",didiStartAddr1.userid
#clear the old and add a new element
didiStartAddr1.Clear()
didiStartAddr1.userid = id
startAddr = didiStartAddr1.astartaddr.add()
startAddr.fromlng = x #string.atof(vec[4])
startAddr.fromlat = y #string.atof(vec[5])
#the last line
if (0 != didiStartAddr1.userid):
key = "start" + str(didiStartAddr1.userid)
value = didiStartAddr1.SerializeToString()
r.set(key, value)
print "set id",didiStartAddr1.userid
print "StartAddrFile lines=",i," num=",num,"count=",count
open_file.close()
end = datetime.datetime.now()
print "endtime=%s",end
print "start Time consume:",(end - start).seconds