def __init__(self, host, db, user, password, ak="DW2CwL3B3271CiVyw7GdBsfR"):
logging.debug("Constructor ak:%s" % ak)
self.baiduAPIService = BaiduMapAPIService(ak)
self.baiduMapDAO = BaiduMapDAO(host, db, user, password)
self.around_facilities_distance = [] # 存放最后要存入数据库的周边设施列表
self.around_data = {} # 存放最后存入数据库的全部信息列表
self.around_facilities_zuobiao = [] # 酒店坐标列表
self.around_facilities_zhoubiansheshibiaozuoliebiao = [] # 周边设施坐标列表
self.facilities_ditance = [] # 存储设施名字对应与酒店间距离的列表
self.maxdistance_and_hotelzuobiao = [] # 存放最远距离设置和坐标
self.hotelname_and_zuobiao = [] #存放酒店的名字和坐标
self.disigeziduan = [] #最后存入数据库的第四个字段
self.bed = {}
# -*- coding:utf-8 -*-
__author__ = 'DreamCathcer'
import traceback
from dao.hotel.TuniuDao import TuniuDAO
from setting import local_hotel_setting
from service.map.baidu.APIService import BaiduMapAPIService
# 配置数据库
dao_setting = local_hotel_setting
dao = TuniuDAO(dao_setting["host"], dao_setting["db"], dao_setting["user"],
dao_setting["password"])
map_service = BaiduMapAPIService("MviPFAcx5I6f1FkRQlq6iTxc")
hotellist = dao.get_hotelinfo()
# 酒店地理编码容器
hotel_location = []
# 遍历酒店信息，取出酒店名称进行地理编码
for i in range(0, len(hotellist)):
geocoding_info = map_service.doGeocoding(hotellist[i][1])
try:
geocoding_info = {
"hotel_name": hotellist[i][1],
"x": geocoding_info["result"]["location"]["lng"],
"y": geocoding_info["result"]["location"]["lat"]
}
def saveListPageInfo(self):
baidu_api_service = BaiduMapAPIService("MviPFAcx5I6f1FkRQlq6iTxc")
old_location_info = self.hotel_dao.get_locations(self._city)
old_baseinfo = list(self.hotel_dao.get_baseinfo(self._city, "途牛"))
# 将基础数据中的if_overtime先假设为都已过时
for i in range(0, len(old_baseinfo)):
old_baseinfo[i] = list(old_baseinfo[i])
old_baseinfo[i][5] = 1
new_locations = []
new_baseinfo = []
update_baseinfo = []
# 遍历将要保存的数据
for item in self.listPageInfo:
location_id = None
# 首先检查该酒店是否已经保存在location表中
for location in old_location_info:
if item["hotel_name"] == location[3]:
location_id = location[0]
break
# 如果没有则插入一条新的记录到location表和baseinfo表
if location_id == None:
location_id = uuid.uuid1()
while 1:
try:
geocoding_info = baidu_api_service.doGeocoding(item["hotel_name"], city=self._city)
break
except:
time.sleep(0.5)
continue
if "result" not in geocoding_info:
print item["hotel_name"] + "error"
continue
trans_location = CoordTransor.bd09togcj02(bd_lon=geocoding_info["result"]["location"]["lng"], bd_lat=geocoding_info["result"]["location"]["lat"])
print trans_location
new_locations.append({
"guid":location_id,
"x": trans_location[1],
"y": trans_location[0],
"hotel_name":item["hotel_name"],
"city":self._city
})
new_baseinfo.append({
"guid":item["guid"],
"url":item["url"],
"location_id":location_id,
"OTA":"途牛",
"comm_num":item["comm_num"],
"if_overtime":0,
"incre_num":item["comm_num"],
})
# 如果存在于location表,则其在baseinfo表中的记录进行更新
else:
for baseinfo in old_baseinfo:
if location_id == baseinfo[2]:
baseinfo[1] = item["url"]
baseinfo[4] = item["comm_num"]
baseinfo[5] = 0
baseinfo[6] = item["comm_num"] - baseinfo[4] if item["comm_num"]-baseinfo[4]>0 else 0
break
for baseinfo in old_baseinfo:
update_baseinfo.append({
"guid":baseinfo[0],
"url":baseinfo[1],
"location_id":baseinfo[2],
"OTA":baseinfo[3],
"comm_num":baseinfo[4],
"if_overtime":baseinfo[5],
"incre_num":baseinfo[6]
})
print len(new_locations)
print len(new_baseinfo)
print len(update_baseinfo)
self.hotel_dao.save_locations(new_locations)
self.hotel_dao.save_baseinfo(new_baseinfo)
self.hotel_dao.update_baseinfo(update_baseinfo)
from util.io.CSVFileUtil import CSVFileUtil
from service.map.baidu.APIService import BaiduMapAPIService
input_file = r'C:\Users\kaipeng\Desktop\rent.csv'
output_file = r"C:\Users\kaipeng\Desktop\rent_geocode.csv"
have_title = True
handle_row_index = 2
def handle_text(text):
return "广州市".decode("utf-8").encode("gbk") + re.sub("[\[\]]", "", text)
if __name__ == "__main__":
csv_file_util = CSVFileUtil()
map_service = BaiduMapAPIService("WBw4kIepZzGp4kH5Gn3r0ACy")
writer = csv.writer(file(output_file, "wb"))
count = 0
for line in csv_file_util.reader(file(input_file)):
count += 1
if have_title and count == 1:
continue
geocoding_info = map_service.doGeocoding(
handle_text(line[handle_row_index]))
try:
coord = str(
geocoding_info["result"]["location"]["lng"]) + ',' + str(
geocoding_info["result"]["location"]["lat"])
line[3] = coord
print "Success:count:%d" % count
except:
import re
from util.io.CSVFileUtil import CSVFileUtil
from service.map.baidu.APIService import BaiduMapAPIService
input_file = r'C:\Users\kaipeng\Desktop\rent.csv'
output_file = r"C:\Users\kaipeng\Desktop\rent_geocode.csv"
have_title = True
handle_row_index = 2
def handle_text(text):
return "广州市".decode("utf-8").encode("gbk")+re.sub("[\[\]]","",text)
if __name__=="__main__":
csv_file_util = CSVFileUtil()
map_service = BaiduMapAPIService("WBw4kIepZzGp4kH5Gn3r0ACy")
writer = csv.writer(file(output_file, "wb"))
count = 0
for line in csv_file_util.reader(file(input_file)):
count += 1
if have_title and count==1:
continue
geocoding_info = map_service.doGeocoding(handle_text(line[handle_row_index]))
try:
coord = str(geocoding_info["result"]["location"]["lng"])+','+str(geocoding_info["result"]["location"]["lat"])
line[3] = coord
print "Success:count:%d"%count
except:
traceback.print_exc()
print "Error:count:%d"%count
continue
# -*- coding:utf-8 -*-
__author__ = 'DreamCathcer'
import traceback
from dao.hotel.TuniuDao import TuniuDAO
from setting import local_hotel_setting
from service.map.baidu.APIService import BaiduMapAPIService
# 配置数据库
dao_setting = local_hotel_setting
dao = TuniuDAO(dao_setting["host"], dao_setting["db"], dao_setting["user"], dao_setting["password"])
map_service = BaiduMapAPIService("MviPFAcx5I6f1FkRQlq6iTxc")
hotellist = dao.get_hotelinfo()
# 酒店地理编码容器
hotel_location = []
# 遍历酒店信息，取出酒店名称进行地理编码
for i in range(0, len(hotellist)):
geocoding_info = map_service.doGeocoding(hotellist[i][1])
try:
geocoding_info = {"hotel_name":hotellist[i][1], "x":geocoding_info["result"]["location"]["lng"], "y":geocoding_info["result"]["location"]["lat"]}
except:
traceback.print_exc()
continue
hotel_location.append(geocoding_info)
print "%d done"%i
# -*- coding:utf-8 -*-
__author__ = 'DreamCathcer'
from service.map.baidu.APIService import BaiduMapAPIService
from util.geo import CoordTransor
baidu_api_service = BaiduMapAPIService("MviPFAcx5I6f1FkRQlq6iTxc")
f = open('sightspot.txt', 'r')
for line in f.readlines():
data = baidu_api_service.doGeocoding(addressText=line.strip(), city='南京')
if "result" in data:
print line + str(
CoordTransor.bd09togcj02(data["result"]["location"]["lng"],
data["result"]["location"]["lat"]))
# print line+str(data["result"]["location"]["lat"])+","+str(data["result"]["location"]["lng"])
# -*- coding:utf-8 -*-
__author__ = 'DreamCathcer'
from service.map.baidu.APIService import BaiduMapAPIService
from util.geo import CoordTransor
baidu_api_service = BaiduMapAPIService("MviPFAcx5I6f1FkRQlq6iTxc")
f = open('sightspot.txt', 'r')
for line in f.readlines():
data = baidu_api_service.doGeocoding(addressText=line.strip(),city='南京')
if "result" in data:
print line+str(CoordTransor.bd09togcj02(data["result"]["location"]["lng"],data["result"]["location"]["lat"]))
# print line+str(data["result"]["location"]["lat"])+","+str(data["result"]["location"]["lng"])
class BaiduMapSnatcherService(object):
def __init__(self, host, db, user, password, ak="DW2CwL3B3271CiVyw7GdBsfR"):
logging.debug("Constructor ak:%s" % ak)
self.baiduAPIService = BaiduMapAPIService(ak)
self.baiduMapDAO = BaiduMapDAO(host, db, user, password)
self.around_facilities_distance = [] # 存放最后要存入数据库的周边设施列表
self.around_data = {} # 存放最后存入数据库的全部信息列表
self.around_facilities_zuobiao = [] # 酒店坐标列表
self.around_facilities_zhoubiansheshibiaozuoliebiao = [] # 周边设施坐标列表
self.facilities_ditance = [] # 存储设施名字对应与酒店间距离的列表
self.maxdistance_and_hotelzuobiao = [] # 存放最远距离设置和坐标
self.hotelname_and_zuobiao = [] #存放酒店的名字和坐标
self.disigeziduan = [] #最后存入数据库的第四个字段
self.bed = {}
# def __del__(self):
# print "... Destructor BaiduMapSnatcherService... %s" % multiprocessing.current_process().name
'''
取出设施距离数据
'''
def Get_around_facilities_data(self):
data = self.baiduMapDAO.get_around_facilities_data()
b = 0 #距离在100以下
c = 0 #距离在100到200
d = 0 #距离在200到300
e = 0 #距离在300到400
f = 0 #距离在400到500
g = 0 #距离在500到600
h = 0 #距离在600到700
I = 0 #距离在700到800
J = 0 #距离在800到900
K = 0 #距离在900到1000
L = 0 #距离在1000以上
for i in data:
a = json.loads(i[0])
if a[1][1] <100:
b = b+1
elif a[1][1] >= 100 and a[1][1] < 200:
c = c+1
elif a[1][1] >= 200 and a[1][1] <300:
d = d+1
elif a[1][1] >= 300 and a[1][1] < 400:
e = e + 1
elif a[1][1] >= 400 and a[1][1] < 500:
f = f + 1
elif a[1][1] >= 500 and a[1][1] < 600:
g = g + 1
elif a[1][1] >= 600 and a[1][1] < 700:
h = h + 1
elif a[1][1] >= 700 and a[1][1] < 800:
I = I + 1
elif a[1][1] >= 800 and a[1][1] < 900:
J = J + 1
elif a[1][1] >= 900 and a[1][1] < 1000:
K = K + 1
elif a[1][1] >= 1000:
L = L + 1
print str(b) + " " + str(c) + " " + str(d) + " " + str(e) + " " + str(f) + " " + str(g) + " " + str(h) + " " + str(I) + " " + str(J) + " " + str(K) + " " + str(L)
'''
从数据库读取周边设施数据
'''
def getdata(self):
Data = self.baiduMapDAO._returnarou()
for data in Data:
if len(data[-1]):
try:
self.manage_around_falities(data[-1],data[2])
self.around_data["hotelname"] = data[2]
self.around_data["facilities_lntandlang"] = self.around_facilities_distance
self.get_maxdistance_facilities()
# self.baiduMapDAO.save_around_facilities(self.around_data)
self.facilities_ditance = []
self.disigeziduan = []
self.maxdistance_and_hotelzuobiao = []
self.around_facilities_zuobiao = []
self.hotelname_and_zuobiao = []
self.around_facilities_distance = []
self.around_facilities_zhoubiansheshibiaozuoliebiao = []
self.around_data = {}
except:
pass
'''
南京市纬度范围31°14″至32°37″
南京市经度范围东经118°22″至119°14″
'''
def manage_around_falities(self,data,hoteldata):
if len(data):
data1 = re.sub(':',',',data)
data2 = re.sub(';',',',data1)
data3 = re.sub(',,',',',data2)
data4 = data3.split(",")
hotel_coordinate_data = self.baiduAPIService.doGeocoding(hoteldata,u"南京市")
hotel_coordinate_x = hotel_coordinate_data["result"]["location"]["lng"]
hotel_coordinate_y = hotel_coordinate_data["result"]["location"]["lat"]
self.around_facilities_zuobiao.append(hotel_coordinate_x)
self.around_facilities_zuobiao.append(hotel_coordinate_y)
self.hotelname_and_zuobiao.append(hoteldata)
self.hotelname_and_zuobiao.append(self.around_facilities_zuobiao)
self.around_facilities_distance.append(self.around_facilities_zuobiao)
# i 是传入的地址名
for i in range(len(data4)):
self.getjingweifu(data4[i],hotel_coordinate_x,hotel_coordinate_y)
self.around_facilities_distance.append(self.around_facilities_zhoubiansheshibiaozuoliebiao)
'''
传入地址名获得经纬度信息
'''
def getjingweifu(self,facilities_data,hotel_coordinate_x,hotel_coordinate_y):
try:
data = self.baiduAPIService.doGeocoding(facilities_data,u"南京市")
if data['result']['location']["lng"] >= 118:
if data['result']['location']["lng"] <= 119.5:
if data['result']['location']["lat"] >=30.5:
if data['result']['location']["lat"] <=32.8:
if len(self.around_facilities_zhoubiansheshibiaozuoliebiao) < 3:
self.around_facilities_zhoubiansheshibiaozuoliebiao.append([data['result']['location']["lng"],data['result']['location']["lat"]])
self.compute(hotel_coordinate_x,hotel_coordinate_y,data['result']['location']["lng"],data['result']['location']["lat"],facilities_data)
except:
pass
'''
计算酒店和设施点的距离
'''
def compute(self,hotel_x,hotel_y,facilities_x,facilities_y,facilities_name):
distance = 6371 * math.acos(math.cos(hotel_y)*math.cos(facilities_y)*math.cos(hotel_x-facilities_x)+math.sin(hotel_y)*math.sin(facilities_y))
print distance
self.facilities_ditance.append({facilities_name:distance})
'''
找出最远距离的设施点
'''
def get_maxdistance_facilities(self):
min = self.facilities_ditance[0].values()
for i in self.facilities_ditance:
if i.values()[0] < min:
min = i.values()[0]
max = min
for i in self.facilities_ditance:
if i.values()[0] > max:
if i.values()[0] < 1500:
max = i.values()[0]
for i in self.facilities_ditance:
for q,w in i.items():
if w == max:
self.maxdistance_and_hotelzuobiao.append(q)
self.maxdistance_and_hotelzuobiao.append(w)
self.disigeziduan.append(self.hotelname_and_zuobiao)
self.disigeziduan.append(self.maxdistance_and_hotelzuobiao)
self.disigeziduan
self.around_data["maxsite"] = self.disigeziduan
break
'''
计算地球上两点的距离
'''
def calcDistance(Lat_A, Lng_A, Lat_B, Lng_B):
ra = 6378.140 # 赤道半径 (km)
rb = 6356.755 # 极半径 (km)
flatten = (ra - rb) / ra # 地球扁率
rad_lat_A = radians(Lat_A)
rad_lng_A = radians(Lng_A)
rad_lat_B = radians(Lat_B)
rad_lng_B = radians(Lng_B)
pA = atan(rb / ra * tan(rad_lat_A))
pB = atan(rb / ra * tan(rad_lat_B))
xx = acos(sin(pA) * sin(pB) + cos(pA) * cos(pB) * cos(rad_lng_A - rad_lng_B))
c1 = (sin(xx) - xx) * (sin(pA) + sin(pB)) ** 2 / cos(xx / 2) ** 2
c2 = (sin(xx) + xx) * (sin(pA) - sin(pB)) ** 2 / sin(xx / 2) ** 2
dr = flatten / 8 * (c1 - c2)
distance = ra * (xx + dr)
return distance
# 抓取Place详情
def fetchPlaceDetail(self, lng0, lat0, lng1, lat1, query):
# 使用矩形范围初始栈
queue = [[lng0, lat0, lng1, lat1]]
# 用以存放返回值为0的矩形范围
zero_queue = []
while len(queue) != 0:
# 取出一个查询范围
range = queue.pop()
# 根据范围进行查询
try:
data = self.baiduAPIService.placeSearch(query=query,
bounds="%lf,%lf,%lf,%lf" % (range[1], range[0], range[3], range[2]))
except Exception, e:
print e
print u"查询数据出错"
queue.append(range)
continue
# 查看数据是否有效
try:
print u"范围查询就结果个数： " + str(len(data['results']))
except:
print u"查询数据返回内容出错"
continue
if data.has_key('results'):
# 如果范围的poi等于20,就切割该范围,并将切割后的子范围置入栈中
if len(data['results']) == 20:
splitX = (range[0] + range[2]) / 2
splitY = (range[1] + range[3]) / 2
if (range[2] - range[0]) < 0.001 or (range[3] - range[1]) < 0.001:
continue
queue.append([range[0], splitY, splitX, range[3]])
queue.append([splitX, splitY, range[2], range[3]])
queue.append([range[0], range[1], splitX, splitY])
queue.append([splitX, range[1], range[2], splitY])
continue
elif len(data['results']) == 0:
zero_queue.append(range)
# 如果查询结果小于20则存储
else:
self.baiduMapDAO.savePlaceDetail(data)
return zero_queue
def saveListPageInfo(self):
print "开始保存"
baidu_api_service = BaiduMapAPIService("MviPFAcx5I6f1FkRQlq6iTxc")
old_location_info = self.hotel_dao.get_locations(self._city)
old_baseinfo = list(
self.hotel_dao.get_baseinfo(self._city, self.__ota_info))
# 将基础数据中的if_overtime先假设为都已过时
for i in range(0, len(old_baseinfo)):
old_baseinfo[i] = list(old_baseinfo[i])
old_baseinfo[i][5] = 1
new_locations = []
new_baseinfo = []
update_baseinfo = []
# 遍历将要保存的数据
for item in self.listPageInfo:
location_id = None
# 首先检查该酒店是否已经保存在location表中
for location in old_location_info:
if item["hotel_name"] == location[3]:
location_id = location[0]
break
# 如果没有则插入一条新的记录到location表中
if location_id is None:
location_id = uuid.uuid1()
while 1:
try:
geocoding_info = baidu_api_service.doGeocoding(
item["address"], city=self._city)
if "result" not in geocoding_info:
geocoding_info = baidu_api_service.doGeocoding(
item["hotel_name"], city=self._city)
break
except:
time.sleep(0.5)
continue
if "result" not in geocoding_info:
print item["hotel_name"] + "error"
continue
trans_location = CoordTransor.bd09togcj02(
bd_lon=geocoding_info["result"]["location"]["lng"],
bd_lat=geocoding_info["result"]["location"]["lat"])
#trans_location = [geocoding_info["result"]["location"]["lng"], geocoding_info["result"]["location"]["lat"]]
#print trans_location
new_locations.append({
"guid": location_id,
"x": trans_location[1],
"y": trans_location[0],
"hotel_name": item["hotel_name"],
"city": self._city
})
if_exist = False
for baseinfo in old_baseinfo:
if location_id == baseinfo[2]:
if_exist = True
baseinfo[1] = item["url"]
baseinfo[6] = int(item["comm_num"]) - int(baseinfo[4])
baseinfo[4] = item["comm_num"]
baseinfo[5] = 0
#baseinfo[6] = item["comm_num"] - baseinfo[4] if item["comm_num"] - baseinfo[4] > 0 else 0
break
if not if_exist:
new_baseinfo.append({
"guid": item["guid"],
"url": item["url"],
"location_id": location_id,
"OTA": self.__ota_info,
"comm_num": item["comm_num"],
"if_overtime": 0,
"incre_num": item["comm_num"],
})
for baseinfo in old_baseinfo:
update_baseinfo.append({
"guid": baseinfo[0],
"url": baseinfo[1],
"location_id": baseinfo[2],
"OTA": baseinfo[3],
"comm_num": baseinfo[4],
"if_overtime": baseinfo[5],
"incre_num": baseinfo[6]
})
print len(new_locations), len(new_baseinfo), len(update_baseinfo)
self.hotel_dao.save_locations(new_locations)
self.hotel_dao.save_baseinfo(new_baseinfo)
self.hotel_dao.update_baseinfo(update_baseinfo)
pass