def other2baidu(lng, lat, maptype):
    if maptype == u'soso':
        lng, lat = _coordtrans.coordtrans('gcj02', 'bd09ll', float(lng),
                                          float(lat))
    elif maptype == u'wgs':
        lng, lat = _coordtrans.coordtrans('wgs84', 'bd09ll', float(lng),
                                          float(lat))
    else:
        lng, lat = float(lng), float(lat)

    return lng, lat
Ejemplo n.º 2
0
def parseHadoopFile(filename, r):
    print "fileName:", filename
    open_file = open(filename, "r")
    i = 0
    num = 0
    setnum = 0
    strCmd = {}
    # pipe = r.pipeline(transaction=False)
    for line in open_file:
        i += 1
        strline = re.sub(" {1,}", " ", line)
        vec = re.split(" |#|\t", strline)
        if 5 != len(vec):
            continue
        num += 1
        setnum += 1
        # transfer to soso coord
        y, x = string.atof(vec[2]), string.atof(vec[1])
        (x, y) = coordtrans("bd09ll", "gcj02", x, y)
        y1, x1 = string.atof(vec[4]), string.atof(vec[3])
        (x1, y1) = coordtrans("bd09ll", "gcj02", x1, y1)
        # print "baidu: ",vec[1],vec[2],vec[3],vec[4]
        # print "soso: ",x,y,x1,y1

        userAddr = userInfo_pb2.UserAddr()
        userAddr.userid = string.atol(vec[0])
        userAddr.homelng = x
        userAddr.homelat = y
        userAddr.corplng = x1
        userAddr.corplat = y1
        key = "addr" + vec[0]
        value = userAddr.SerializeToString()
        # r.set(key, value)
        # print key,value
        strCmd[key] = value
        if setnum >= 200:
            # print "setnum=",setnum,strCmd
            r.mset(strCmd)
            strCmd.clear()
            setnum = 0
    if strCmd:
        # print "last setnum=%d"%(setnum)
        r.mset(strCmd)
        strCmd.clear()
        setnum = 0
    print "hadoopFile,", filename, "lines=", i, "num=", num
    open_file.close()
Ejemplo n.º 3
0
def parseMultiDestFile(filename, r):
	print "fileName:",filename
	open_file = open(filename, 'r')
	i = 0
	num = 0
	count = 0
	didiDestAddr1 = userInfo_pb2.DidiDestAddr()
	didiDestAddr1.Clear()
	for line in open_file:
		i += 1
		vec = line.split(',')
		if (len(vec) < 7):
			#print "--------------------------less than 7",len(vec)
			continue
		first = int(vec[0])
		if (0 != first):
			num += 1
			#print vec[0],vec[4],vec[5]
			id = string.atol(vec[0])

			#coordtrans: from baidu to soso 
			y,x = string.atof(vec[5]), string.atof(vec[4])
			(x, y) = coordtrans('bd09ll', 'gcj02', x, y)
			#print "befor:x=%lf,y=%lf"%(string.atof(vec[4]), string.atof(vec[5]))
			#print "after:x=%lf,y=%lf"%(x, y)

			if (1 == num):
				didiDestAddr1.userid = id
				destAddr = didiDestAddr1.adestaddr.add() 
				destAddr.tolng = x #string.atof(vec[4])
				destAddr.tolat = y #string.atof(vec[5])
			else:
				if (didiDestAddr1.userid == id):
					destAddr = didiDestAddr1.adestaddr.add() 
					destAddr.tolng = x #string.atof(vec[4])
					destAddr.tolat = y #string.atof(vec[5])
					#print "repeate id",id
				else:
					count += 1
					key = "dest" + str(didiDestAddr1.userid)
					value = didiDestAddr1.SerializeToString()
					r.set(key, value)
					print "set id",didiDestAddr1.userid

					#clear the old and add a new element
					didiDestAddr1.Clear()
					didiDestAddr1.userid = id
					destAddr = didiDestAddr1.adestaddr.add() 
					destAddr.tolng = x #string.atof(vec[4])
					destAddr.tolat = y #string.atof(vec[5])
	#the last line				
	if (0 != didiDestAddr1.userid):
		key = "dest" + str(didiDestAddr1.userid)
		value = didiDestAddr1.SerializeToString()
		r.set(key, value)
		print "set id",didiDestAddr1.userid

	print "DestAddrFile lines=",i," num=",num,"count=",count
	open_file.close()
    line = sys.stdin.readline();
    if line == "" :
        break;
    i += 1
    vec = line.split(',')
    if (len(vec) != 7):
        #print "--------------------------less than 7",len(vec)
        continue
    first = int(vec[0])
    if (0 != first):
        num += 1
        id = string.atol(vec[0])

        #coordtrans: from baidu to soso 
        y,x = string.atof(vec[5]), string.atof(vec[4])
        (x, y) = coordtrans('bd09ll', 'gcj02', x, y)
        #print "befor:x=%lf,y=%lf"%(string.atof(vec[4]), string.atof(vec[5]))
        #print "after:x=%lf,y=%lf"%(x, y)

        if (1 == num):
            didiDestAddr1.userid = id
            destAddr = didiDestAddr1.adestaddr.add() 
            destAddr.tolng = x #string.atof(vec[4])
            destAddr.tolat = y #string.atof(vec[5])
        else:
            if (didiDestAddr1.userid == id):
                destAddr = didiDestAddr1.adestaddr.add() 
                destAddr.tolng = x #string.atof(vec[4])
                destAddr.tolat = y #string.atof(vec[5])
            else:
                count += 1
Ejemplo n.º 5
0
def parseMultiStartFile(filename, r):
	start = datetime.datetime.now()
	print "starttime=%s",start

	print "fileName:",filename
	open_file = open(filename, 'r')
	i = 0
	num = 0
	count = 0
	didiStartAddr1 = userInfo_pb2.DidiStartAddr()
	didiStartAddr1.Clear()
	for line in open_file:
	#for line in open_file.readlines():
		i += 1
		vec = line.split(',')
		if (7 != len(vec)):
			#print "-----less than 7",len(vec)
			continue
		first = int(vec[0])
		if (0 != first):
			num += 1
			#print vec[0],vec[4],vec[5]
			id = string.atol(vec[0])

			#coordtrans: from baidu to soso 
			y,x = string.atof(vec[5]), string.atof(vec[4])
			(x, y) = coordtrans('bd09ll', 'gcj02', x, y)

			if (1 == num):
				didiStartAddr1.userid = id
				startAddr = didiStartAddr1.astartaddr.add() 
				startAddr.fromlng = x #string.atof(vec[4])
				startAddr.fromlat = y #string.atof(vec[5])
			else:
				if (didiStartAddr1.userid == id):
					startAddr = didiStartAddr1.astartaddr.add() 
					startAddr.fromlng = x #string.atof(vec[4])
					startAddr.fromlat = y #string.atof(vec[5])
					#print "repeate id",id
				else:
					count += 1
					key = "start" + str(didiStartAddr1.userid)
					value = didiStartAddr1.SerializeToString()
					r.set(key, value)
					print "set id",didiStartAddr1.userid

					#clear the old and add a new element
					didiStartAddr1.Clear()
					didiStartAddr1.userid = id
					startAddr = didiStartAddr1.astartaddr.add() 
					startAddr.fromlng = x #string.atof(vec[4])
					startAddr.fromlat = y #string.atof(vec[5])
	#the last line				
	if (0 != didiStartAddr1.userid):
		key = "start" + str(didiStartAddr1.userid)
		value = didiStartAddr1.SerializeToString()
		r.set(key, value)
		print "set id",didiStartAddr1.userid

        print "StartAddrFile lines=",i," num=",num,"count=",count
	open_file.close()

	end = datetime.datetime.now()
	print "endtime=%s",end
	print "start Time consume:",(end - start).seconds