class TuniuCatcher(object):
def __init__(self):
self._city = None
self.__ota_info = "途牛"
self.tuniu_api_client = TuniuAPIClient()
self.hotel_dao = HotelDAO(dao_setting["host"], dao_setting["db"], dao_setting["user"], dao_setting["password"])
def setCity(self, city):
self._city = city
def getHotelList(self, city_code):
if self._city == None:
print "未设置城市,请先使用setCity方法"
return
hotel_list = []
page_index = 1
page_amount =10000
while page_index <= page_amount - 1:
try:
page_data = self.tuniu_api_client.get_hotel_list(page_index, city_code)
# 接口返回的酒店数不稳定，所以爬取页数以最小数为准
if page_amount > page_data["data"]["total"] / 20:
page_amount = page_data["data"]["total"] / 20
print "page_amount=%d"%page_amount
hotel_list.extend(page_data["data"]["list"])
print "Page_%d Success"%page_index
time.sleep(5)
page_index += 1
except:
print "Page_%d Fail"%page_index
continue
return hotel_list
def saveHolteList(self, hotel_list):
old_location_info = self.hotel_dao.get_locations(self._city)
old_baseinfo = list(self.hotel_dao.get_baseinfo(self._city, self.__ota_info))
# 将基础数据中的if_overtime先假设为都已过时
for i in range(0, len(old_baseinfo)):
old_baseinfo[i] = list(old_baseinfo[i])
old_baseinfo[i][5] = 1
new_locations = []
new_baseinfo = []
update_baseinfo = []
# 遍历将要保存的数据
for item in hotel_list:
location_id = None
# 首先检查该酒店是否已经保存在location表中
for location in old_location_info:
if item["name"] == location[3]:
location_id = location[0]
break
# 如果没有则插入一条新的记录到location表中
if location_id is None:
location_id = uuid.uuid1()
trans_location = CoordTransor.gcj02towgs84(lng=float(item["pos"]["lng"]), lat=float(item["pos"]["lat"]))
new_locations.append({
"guid": location_id,
"x": trans_location[1],
"y": trans_location[0],
"hotel_name": item["name"],
"city": self._city,
"address": item["address"]
})
# 根据location的id号到baseinfo表中查询
# 如果已经存于表中，则更新该条数据
# 如果没有，则插入一条新的数据
if_exist = False
for baseinfo in old_baseinfo:
if location_id == baseinfo[2]:
if_exist = True
baseinfo[1] = item["url"]
baseinfo[4] = item["remarkCount"]
baseinfo[5] = 0
baseinfo[6] = int(item["remarkCount"]) - int(baseinfo[4]) if int(item["remarkCount"]) - int(baseinfo[4]) > 0 else 0
baseinfo[7] = item["snapshot"]
baseinfo[8] = item["id"]
break
if not if_exist:
new_baseinfo.append({
"guid": uuid.uuid1(),
"url": item["url"],
"location_id": location_id,
"OTA": self.__ota_info,
"comm_num": item["remarkCount"],
"if_overtime": 0,
"incre_num": item["remarkCount"],
"img": item["snapshot"],
"id_in_ota": item["id"]
})
for baseinfo in old_baseinfo:
update_baseinfo.append({
"guid": baseinfo[0],
"url": baseinfo[1],
"location_id": baseinfo[2],
"OTA": baseinfo[3],
"comm_num": baseinfo[4],
"if_overtime": baseinfo[5],
"incre_num": baseinfo[6],
"img": baseinfo[7],
"id_in_ota": baseinfo[8]
})
print len(new_locations), len(new_baseinfo), len(update_baseinfo)
self.hotel_dao.save_locations(new_locations)
self.hotel_dao.save_baseinfo(new_baseinfo)
self.hotel_dao.update_baseinfo(update_baseinfo)
class XiechengDriverService(HotelService):
def __init__(self):
HotelService.__init__(self)
# 携程dao
self.xiechengDao = xiechengDAO(dao_setting["host"], dao_setting["db"],
dao_setting["user"],
dao_setting["password"])
# 酒店dao
self.hotel_dao = HotelDAO(dao_setting["host"], dao_setting["db"],
dao_setting["user"], dao_setting["password"])
# 存放列表页数据
self.listPageInfo = []
# 存放酒店详情数据
self.hotelItem = {}
# 存放酒店评论数据
self.commList = []
# 存储床价信息
self.bed = {}
# 当前ota名称
self.__ota_info = "携程"
def crawlListPage(self):
self.openPage(
"http://hotels.ctrip.com/hotel/nanjing12#ctm_ref=hod_hp_sb_lst")
self.driver.implicitly_wait(10)
# 单页循环次数
loopNum = 0
# 标识当前页面是否已经爬取：False为未处理，反之为已处理
ifHandle = False
# 获取总页面数
pageNum = 140
while (pageNum >= 1):
# 循环次数加1
loopNum = loopNum + 1
# 到达页面90%处
# js="var q=document.documentElement.scrollTop=9600"
# self.driver.execute_script(js)
self.driver.find_element_by_tag_name("body").send_keys(Keys.END)
self.driver.find_element_by_tag_name("body").send_keys(
Keys.PAGE_UP)
# 当页面中出现“返前价”字样时，爬取页面并跳转到下一页
if u"收藏" in self.driver.page_source:
# 对未解析过的页面进行解析
if ifHandle == False:
self.__crawllianjie(self.driver.page_source)
print u"获取酒店数为：%d" % len(self.listPageInfo)
ifHandle = True
# 跳转到下一页
try:
if u"下一页" in self.driver.page_source:
self.driver.find_element_by_partial_link_text(
u"下一页").click()
#self.driver.find_element_by_xpath("//a[@class='c_down']").click()
pageNum = pageNum - 1
# 处理标识重新置为未处理
ifHandle = False
# 单页循环次数置为零
loopNum = 0
time.sleep(random.uniform(3, 6))
print u"页数：" + str(pageNum)
except:
print "error happen at clicking of nextpage"
# 如果单页循环次数不为零，说明没有跳转到下一页
if loopNum != 0:
# 循环次数较大的情况下（此处预定为15次）说明页面可能加载失败，跳出循环，否则继续循环获取
if loopNum < 15:
time.sleep(3)
continue
else:
break
return False if pageNum > 1 else True
# 爬取页面链接
def __crawllianjie(self, page_sourse):
response = HtmlResponse(url="my HTML string",
body=page_sourse,
encoding="utf-8")
hotel_list = response.xpath("//div[@class='searchresult_list ']/ul")
for hotel in hotel_list:
url = hotel.xpath(
"li[@class='searchresult_info_name']/h2/a/@href").extract()[0]
address = hotel.xpath(
"li[@class='searchresult_info_name']/p[@class='searchresult_htladdress']/text()"
).extract()[0]
commnum = hotel.xpath(
"li[@class='searchresult_info_judge ']/div/a/span[@class='hotel_judgement']/text()"
).extract()
if len(commnum):
commnum = re.sub('\D', '', commnum[0])
commnum = commnum if len(commnum) > 0 else 0
else:
commnum = 0
name = hotel.xpath(
"li[@class='searchresult_info_name']/h2/a/text()").extract()[0]
self.listPageInfo.append({
"guid": uuid.uuid1(),
"url": url,
"hotel_name": name,
"OTA": self.__ota_info,
"comm_num": int(commnum),
"address": address
})
'''
保存爬取的酒店列表页数据
'''
def saveListPageInfo(self):
baidu_api_service = BaiduMapAPIService("MviPFAcx5I6f1FkRQlq6iTxc")
old_location_info = self.hotel_dao.get_locations(self._city)
old_baseinfo = list(
self.hotel_dao.get_baseinfo(self._city, self.__ota_info))
# 将基础数据中的if_overtime先假设为都已过时
for i in range(0, len(old_baseinfo)):
old_baseinfo[i] = list(old_baseinfo[i])
old_baseinfo[i][5] = 1
new_locations = []
new_baseinfo = []
update_baseinfo = []
# 遍历将要保存的数据
for item in self.listPageInfo:
location_id = None
# 首先检查该酒店是否已经保存在location表中
for location in old_location_info:
if item["hotel_name"] == location[3]:
location_id = location[0]
break
# 如果没有则插入一条新的记录到location表中
if location_id == None:
location_id = uuid.uuid1()
geocoding_info = None
while 1:
try:
geocoding_info = baidu_api_service.doGeocoding(
item["address"], city=self._city)
break
except:
time.sleep(0.5)
continue
if "result" not in geocoding_info:
print item["hotel_name"] + "error"
continue
trans_location = CoordTransor.bd09togcj02(
bd_lon=geocoding_info["result"]["location"]["lng"],
bd_lat=geocoding_info["result"]["location"]["lat"])
print trans_location
new_locations.append({
"guid": location_id,
"x": trans_location[1],
"y": trans_location[0],
"hotel_name": item["hotel_name"],
"city": self._city,
"address": item["address"]
})
# 根据location的id号到baseinfo表中查询
# 如果已经存于表中，则更新该条数据
# 如果没有，则插入一条新的数据
if_exist = False
for baseinfo in old_baseinfo:
if location_id == baseinfo[2]:
if_exist = True
baseinfo[1] = item["url"]
baseinfo[4] = item["comm_num"]
baseinfo[5] = 0
baseinfo[6] = item["comm_num"] - baseinfo[
4] if item["comm_num"] - baseinfo[4] > 0 else 0
break
if not if_exist:
new_baseinfo.append({
"guid": item["guid"],
"url": item["url"],
"location_id": location_id,
"OTA": self.__ota_info,
"comm_num": item["comm_num"],
"if_overtime": 0,
"incre_num": item["comm_num"],
})
for baseinfo in old_baseinfo:
update_baseinfo.append({
"guid": baseinfo[0],
"url": baseinfo[1],
"location_id": baseinfo[2],
"OTA": baseinfo[3],
"comm_num": baseinfo[4],
"if_overtime": baseinfo[5],
"incre_num": baseinfo[6]
})
print len(new_locations)
print len(new_baseinfo)
print len(update_baseinfo)
self.hotel_dao.save_locations(new_locations)
self.hotel_dao.save_baseinfo(new_baseinfo)
self.hotel_dao.update_baseinfo(update_baseinfo)
#self.dao.saveListPageInfo(self.listPageInfo)
def depose(self):
self.driver.close()