return json.dumps({'grade_list': p( 'div:nth-child(3) > div.review-filter > ul.ta-list.clearfix > li.taService').text().split('\n'), 'comment_num_list': p( 'div:nth-child(3) > div.review-filter > ul.filter-list.clearfix').text().split('\n')[1:]}, ensure_ascii=False) fl_shop2 = Fieldlist( Field(fieldname=FieldName.SHOP_ADDRESS, css_selector='#hotel-page > div > div.hotel-box.hotel-baseinfo > div.info > div.base > p.address', offset=6, try_times=10, pause_time=5), Field(fieldname=FieldName.SHOP_ROOM_RECOMMEND_ALL, css_selector='#J_RoomList', attr='innerHTML', filter_func=get_room_all), Field(fieldname=FieldName.SHOP_INTRO, css_selector='#hotel-desc', attr='innerHTML', filter_func=get_shop_intro), Field(fieldname=FieldName.SHOP_FACILITIES, css_selector='#hotel-facility', attr='innerHTML', filter_func=get_shop_facility), Field(fieldname=FieldName.SHOP_TRAFFIC, css_selector='#rich-map-wrap', attr='innerHTML', filter_func=get_shop_traffic), Field(fieldname=FieldName.SHOP_STATISTICS, css_selector='#hotel-review', attr='innerHTML', filter_func=get_shop_statistics), ) page_shop_1 = Page(name='飞猪酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#J_List > div'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection)) page_shop_2 = Page(name='飞猪酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div > div.row-center > div > h5 > a'),mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) class FliggyHotelSpider(TravelDriver): def page_shop_2_func(self): try: self.move_to_element_by_css_selector(css_selector='#rich-map-wrap > div.J_RichCon > div.tabs > ul') self.vertical_scroll_by(offset=-200) for i in self.until_presence_of_all_elements_located_by_css_selector(css_selector='#rich-map-wrap > div.J_RichCon > div.tabs > ul > li.J_Tab'): i.click() except Exception: self.error_log(e='找不到元素') time.sleep(3)
Field(fieldname=FieldName.SHOP_IMG, css_selector=' div > div.flt1 > a > img', attr='src',is_info=True), Field(fieldname=FieldName.SHOP_ADDRESS, css_selector='div > div.ct-text > ul > li:nth-child(1) > a', is_info=True), Field(fieldname=FieldName.SHOP_GRADE,css_selector='',filter_func=get_shop_grade), #正则表达式不一样 Field(fieldname=FieldName.SHOP_COMMENT_NUM,css_selector='div > div.ct-text > ul > li:nth-child(2) > a',filter_func=get_comment_num, is_info=True), Field(fieldname=FieldName.SHOP_FEATURE, css_selector='div > div.ct-text > p',is_info=True), Field(fieldname=FieldName.SHOP_PRICE,css_selector= '',filter_func=get_shop_price, is_info=True) ) fl_shop2 = Fieldlist( ) page_shop_1 = Page(name='马蜂窝景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#_j_search_result_left > div:nth-child(1) > div > ul > li',), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection),is_save=True) page_shop_2 = Page() page_shop_2 = Page(name='马蜂窝景点店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div > div.ct-text > h3 > a'), mongodb=Mongodb(db=TravelDriver.db,collection=TravelDriver.shop_collection)) def get_comment_grade(self,_str): return str(_str[-1]) def get_comment_time(self,_str): #时间格式统一为2018-12-08 return _str[0:10] def get_comment_year(self,_str): time = _str[0:10] return time[0:4];
Field(fieldname=FieldName.SHOP_GRADE,css_selector='div > div.hotel-brief.fl > div.satisfaction > span.highlight',is_info=True), #正则表达式不一样 #mainHotelLeft > div:nth-child(2) > div > div:nth-child(2) > ul > li:nth-child(2) Field(fieldname=FieldName.SHOP_COMMENT_NUM,css_selector='div > div.hotel-brief.fl > div.comment > a > span', is_info=True), Field(fieldname=FieldName.SHOP_FEATURE, css_selector='div > div.hotel-info.fl > div.nameAndIcon > span.decorate_year',is_info=True), Field(fieldname=FieldName.SHOP_PRICE, css_selector='div > div.hotel-brief.fl > div.startPrice > span.digit', is_info=True), Field(fieldname=FieldName.SHOP_RATE, css_selector='',filter_func=get_shop_rate, is_info=True), ) fl_shop2 = Fieldlist() page_shop_1 = Page(name='途牛酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#main > div.hotel-list > div'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection),is_save=True) page_shop_2 = Page() page_shop_2 = Page(name='途牛酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div > div.hotel-info.fl > div.nameAndIcon > a'), mongodb=Mongodb(db=TravelDriver.db,collection=TravelDriver.shop_collection)) def get_comment_user_name(self, _str): return _str.split(' ')[0] def get_comment_time(self, _str): return re.findall(r'([\d]{4}-[\d]{2}-[\d]{2} [\d]{2}:[\d]{2})',_str)[0] def get_comment_grade(self,_str): #判断如果含有好 高 if ('好' in _str) and ('但是' in _str) == False: return str("5.0") elif ('好' in _str) and ('但是' in _str) == True:
'div > div > div.clrfix > div.item_hotel_info > div.item_hotel_bsinfo > table > tbody > tr > td.item_hotel_name > div > div.level.levelmargin > a.level_score.js_list_score > strong', is_info=True), Field(fieldname=FieldName.SHOP_RATE, css_selector='', filter_func=get_shop_rate, is_info=True), Field(fieldname=FieldName.SHOP_FEATURE, filter_func=get_shop_feature, css_selector='', is_info=True)) fl_shop2 = Fieldlist() page_shop_1 = Page(name='去哪儿酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector='#jxContentPanel > div', ), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) page_shop_2 = Page( name='去哪儿酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='a.e_title.js_list_name'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection)) def get_comment_grade(self, _str): return str(_str[-1])
is_info=True), #无shop_feature Field(fieldname=FieldName.SHOP_FEATURE, css_selector='', filter_func=get_shop_feature, is_info=True), Field(fieldname=FieldName.SHOP_RATE, css_selector='', filter_func=get_shop_rate, is_info=True), ) page_shop_1 = Page( name='去哪儿景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector= '#main-page > div.mp-main > div:nth-child(2) > ul > li'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) fl_shop2 = Fieldlist( Field( fieldname=FieldName.SHOP_NAME, css_selector= '#main-page > div.mp-main > div.mp-headfigure > div.mp-headfeagure-info > div' ), Field( fieldname=FieldName.SHOP_COMMENT_URL, css_selector= '#main-page > div.mp-main > div.mp-baseinfo > div.mpg-flexbox.mp-flex-card > div:nth-child(1) > a',
#\33 6822720 > div:nth-child(1) > div Field(fieldname=FieldName.SHOP_IMG, css_selector='a > div.img-container.lazy-img-box.fl > img', attr='src', is_info=True), Field(fieldname=FieldName.SHOP_ADDRESS, css_selector= '',filter_func=get_shop_address, is_info=True), #这里应该做一个转换 #\34 187 > div:nth-child(2) > div:nth-child(3) > div > div:nth-child(2) > span:nth-child(1) Field(fieldname=FieldName.SHOP_GRADE,css_selector='',filter_func=get_shop_grade, is_info=True), #正则表达式的使用有问题 Field(fieldname=FieldName.SHOP_COMMENT_NUM,css_selector='a > div.search-scenic-content > div.search-scenic-wrapper > div.search-scenic-detail > p',is_info=True), #无shop_feature Field(fieldname=FieldName.SHOP_FEATURE, css_selector='',filter_func=get_shop_feature, is_info=True), Field(fieldname=FieldName.SHOP_RATE,css_selector='',filter_func=get_shop_rate, is_info=True), Field(fieldname=FieldName.SHOP_COMMENT_URL,css_selector='a',attr='href',filter_func=get_shop_comment_url, is_info=True) ) page_shop_1 = Page(name='途牛景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#search-container > section > div > ul > li'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) fl_shop2 = Fieldlist( Field(fieldname=FieldName.SHOP_NAME, css_selector='#main-page > div.mp-main > div.mp-headfigure > div.mp-headfeagure-info > div'), Field(fieldname=FieldName.SHOP_COMMENT_URL, css_selector='#main-page > div.mp-main > div.mp-baseinfo > div.mpg-flexbox.mp-flex-card > div:nth-child(1) > a', attr='href', is_info=True) ) page_shop_2 = Page(name='途牛景点店铺详情页面', fieldlist=fl_shop2) def get_comment_user_name(self,_str): comment_user_name = _str.split(' ')[0]; return comment_user_name;
Field(fieldname=FieldName.SHOP_NAME,css_selector=' div > div.h_info > div.h_info_text > div.h_info_base > p.h_info_b1 > a > span.info_cn',attr='innerHTML', is_info=True), Field(fieldname=FieldName.SHOP_URL,css_selector='div > div.h_info_text > div.h_info_base > p.h_info_b1 > a',attr='href',is_info=True), Field(fieldname=FieldName.SHOP_IMG, css_selector='div.h_info_pic > a > img', attr='big-src',is_info=True), #有些问题 Field(fieldname=FieldName.SHOP_ADDRESS, css_selector='div > div.h_info_text > div.h_info_base > p.h_info_b2',is_info=True), Field(fieldname=FieldName.SHOP_PRICE,css_selector='div > div.h_info_text > div.h_info_pri > p:nth-child(1) > a > span.h_pri_num',is_info=True), #稍许有些问题 Field(fieldname=FieldName.SHOP_COMMENT_NUM,css_selector='div > div.h_info_text > div.h_info_comt > a > span.c555.block.mt5'), Field(fieldname=FieldName.SHOP_GRADE, css_selector=' div > div.h_info_text > div.h_info_comt > a > span.h_info_comt_bg > i.c37e',is_info=True), Field(fieldname=FieldName.SHOP_RATE,css_selector='',filter_func=get_shop_rate, is_info=True), Field(fieldname=FieldName.SHOP_FEATURE,css_selector='',filter_func=get_shop_feature, is_info=True) ) fl_shop2 = Fieldlist() page_shop_1 = Page(name='艺龙酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#hotelContainer > div > div'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection),is_save=True) # page_shop_2 = Page() # page_shop_2 = Page(name='艺龙酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div > div.h_info_text > div.h_info_base > p.h_info_b1 > a'), mongodb=Mongodb(db=TravelDriver.db,collection=TravelDriver.shop_collection)) fl_comment1 = Fieldlist( Field(fieldname=FieldName.COMMENT_USER_NAME, css_selector=' div.cmt_userinfo > div > p.cmt_un',is_info=True), Field(fieldname=FieldName.COMMENT_TIME, css_selector='div.cmt_info_mn > div > div.if_hd_r > span.cmt_con_time', is_info=True), Field(fieldname=FieldName.SHOP_NAME, css_selector='body > div.hdetail_rela_wrap > div > div.hrela_ns_wrap.clearfix > div.hdetail_main.hrela_name > div > h1', is_isolated=True, is_info=True), Field(fieldname=FieldName.COMMENT_CONTENT, css_selector='div.cmt_info_mn > p.cmt_txt',is_info=True), #有问题 Field(fieldname=FieldName.COMMENT_SCORE, css_selector='div.cmt_info_mn > div > div.if_hd > b',is_info=True), ) page_comment_1 = Page(name='艺龙酒店评论列表', fieldlist=fl_comment1, listcssselector=ListCssSelector(list_css_selector='#review > ul > li'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.comments_collection), is_save=True)
' div.product-regular.clearfix > div.product-section > dl:nth-child(6) > dd > div' ), Field( fieldname=FieldName.SHOP_GRADE, css_selector= 'div.product-regular.clearfix > div.product-info > ul > li:nth-child(1) > b', filter_func=get_shop_grade, is_info=True), ) fl_shop2 = Fieldlist() page_shop_1 = Page(name='驴妈妈景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector=' div.product-list > div', item_css_selector='div'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) page_shop_2 = Page() page_shop_2 = Page( name='驴妈妈景点店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup( click_css_selector= 'div.product-regular.clearfix > div.product-section > h3 > a'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=False)
# fl_shop2 = Fieldlist( # Field(fieldname=FieldName.SHOP_PRICE, css_selector='div.piao_wrap redraw > div.mp-description.pngfix > div.mp-description-detail > div.mp-description-price > span.mp-description-qunar-price > em', pause_time=3, is_focus=True, is_info=True), # #评论次数还要改善 # Field(fieldname=FieldName.SHOP_TIME,css_selector="#mp-charact > div > div.mp-charact-time > div.mp-charact-content > div.mp-charact-desc > p"), # Field(fieldname=FieldName.SHOP_COMMENT_NUM, css_selector='div.piao_wrap redraw > div.mp-description.pngfix > div.mp-description-detail > div.mp-description-comments > span.mp-description-commentCount > a', is_focus=True), # Field(fieldname=FieldName.SHOP_SERVICE,css_selector='#root > div > div > div > div > div:nth-child(3) > div.main-bd > div > div.brief-box.clearfix > div.brief-right > ul > li.promise',attr='innerHTML', filter_func=get_shop_service, is_focus=True), # Field(fieldname=FieldName.SHOP_TICKET, css_selector='div.mp-tickets',attr='innerHTML', filter_func=get_shop_ticket, is_focus=True), # Field(fieldname=FieldName.SHOP_INFO, css_selector='#mp-charact > div > div.mp-charact-intro > div.mp-charact-desc > p', attr='innerHTML', filter_func=get_shop_info, is_focus=True) # ) # page_shop_1 = Page(name='去哪儿景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector='#search-list > div', item_css_selector='div'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection)) #page_shop_2 = Page(name='去哪儿景点店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div.sight_item_detail.clrfix > div.sight_item_about > h3 > a'), mongodb=Mongodb(db=TravelDriver.db,collection=TravelDriver.shop_collection), is_save=True) # # def get_comment_user_name(self, _str): # return _str.split(' ')[0] # # def get_comment_time(self, _str): # return re.findall(r'([\d]{4}-[\d]{2}-[\d]{2} [\d]{2}:[\d]{2})',_str)[0] # # fl_comment1 = Fieldlist( # Field(fieldname=FieldName.COMMENT_USER_NAME, css_selector='span.mp-comments-username', filter_func=get_comment_user_name), # Field(fieldname=FieldName.COMMENT_TIME, css_selector='span.mp-comments-time', filter_func=get_comment_time),
#img还有些许问题 #\33 6822720 > div:nth-child(1) > div Field(fieldname=FieldName.SHOP_IMG, css_selector='a > div.img-container.lazy-img-box.fl > img', attr='src', is_info=True), Field(fieldname=FieldName.SHOP_ADDRESS, css_selector= '',filter_func=get_shop_address, is_info=True), #这里应该做一个转换 #\34 187 > div:nth-child(2) > div:nth-child(3) > div > div:nth-child(2) > span:nth-child(1) Field(fieldname=FieldName.SHOP_GRADE,css_selector='',filter_func=get_shop_grade, is_info=True), #正则表达式的使用有问题 Field(fieldname=FieldName.SHOP_COMMENT_NUM,css_selector='a > div.search-scenic-content > div.search-scenic-wrapper > div.search-scenic-detail > p',is_info=True), #无shop_feature Field(fieldname=FieldName.SHOP_FEATURE, css_selector='',filter_func=get_shop_feature, is_info=True), Field(fieldname=FieldName.SHOP_RATE,css_selector='',filter_func=get_shop_rate, is_info=True) ) page_shop_1 = Page(name='途牛景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#search-container > section > div > ul > li'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) def get_comment_grade(self,_str): doc = pq(_str) if (doc('.star-active').length) == 3: return "5" elif (doc('.star-active').length) == 2: return "2.5" else: return "0" def get_comment_year(self,_str): time =_str[0:10] return time[0:4]; def get_comment_season(self, _str):
is_debug='True', filter_func=get_shop_grade, is_info=True), ) # fl_shop2 = Fieldlist( # Field(fieldname=FieldName.SHOP_ADDRESS, css_selector='div.container > div.hotel-intro > div.intro-hd > div.location > span', attr='title', offset=6, try_times=10, pause_time=1), # Field(fieldname=FieldName.SHOP_ROOM_RECOMMEND_ALL, css_selector='#_j_booking_info', attr='innerHTML', filter_func=get_shop_room_all, offset=6, try_times=10, pause_time=2), # Field(fieldname=FieldName.SHOP_TRAFFIC, css_selector='#_j_map_poi_list > div.bd', attr='innerHTML', filter_func=get_shop_traffic, offset=6, try_times=10, pause_time=1), # Field(fieldname=FieldName.SHOP_FACILITIES, css_selector='#_j_hotel_info', attr='innerHTML', filter_func=get_shop_facilities, offset=6, try_times=10, pause_time=1), # Field(fieldname=FieldName.SHOP_STATISTICS, css_selector='#_j_comment', attr='innerHTML', filter_func=get_shop_stattistics), # ) page_shop_1 = Page(name='马蜂窝酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector='#_j_hotel_list > div.hotel-item'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection)) # page_shop_2 = Page(name='马蜂窝酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div.hotel-pic > a'),mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) class MafengwoHotelSpider(TravelDriver): def get_shop_info(self): try: shop_data_list = self.from_page_get_data_list(page=page_shop_1) # self.from_page_add_data_to_data_list(page=page_shop_2, data_list=shop_data_list, pre_page=page_shop_1) except Exception as e: self.error_log(e=e) def get_shop_info_list(self):
pause_time=5), Field(fieldname=FieldName.SHOP_INTRO, css_selector='#hotelIntroduction > div.hotel_introduction_body'), Field(fieldname=FieldName.SHOP_TRAFFIC, css_selector='#hotelTraffic', attr='innerHTML', filter_func=get_shop_traffic), Field(fieldname=FieldName.SHOP_STATISTICS, css_selector='#hotelUserComment', attr='innerHTML', filter_func=get_shop_statistics), ) page_shop_1 = Page(name='途牛酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector='#main > div.hotel-list > div'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection)) page_shop_2 = Page( name='途牛酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup( click_css_selector='div.hotel-brief.fl > div.hotelDetail > a'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) class TuniuHotelSpider(TravelDriver): def page_shop_2_func(self):
tag_list.append(i.text()) statistics.setdefault('tag_list', tag_list) comment_num_list = [] for i in p('dl.rank').items('dd'): comment_num_list.append({re.sub(r'[^\u4e00-\u9fa5]*', '', i.text()): re.sub(r'[^\d]*', '', i.text())}) statistics.setdefault('comment_num_list', comment_num_list) return json.dumps(statistics, ensure_ascii=False) fl_shop2 = Fieldlist( Field(fieldname=FieldName.SHOP_ROOM_RECOMMEND_ALL, css_selector='div.m-room-tools-bd.js-roomtool-rooms.caculate-price', attr='innerHTML', filter_func=get_room_all, pause_time=5), Field(fieldname=FieldName.SHOP_TRAFFIC, css_selector='#js-neighbor', attr='innerHTML', filter_func=get_shop_traffic, pause_time=1), Field(fieldname=FieldName.SHOP_FACILITIES, css_selector='#descContent', attr='innerHTML', filter_func=get_shop_facilities), Field(fieldname=FieldName.SHOP_STATISTICS, css_selector='#comment_main > div > div.b_ugcheader > div.b_ugcfilter', attr='innerHTML', filter_func=get_shop_statistics), ) page_shop_1 = Page(name='去哪儿酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='div.b_result_box.js_list_block.b_result_commentbox'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection)) page_shop_2 = Page(name='去哪儿酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='a.e_title.js_list_name'),mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) class QunarHotelSpider(TravelDriver): def page_shop_2_func(self): try: for i in self.until_presence_of_all_elements_located_by_partial_link_text(link_text='查看其他'): i.click() except Exception: self.error_log(e='找不到元素') try: for i in self.until_presence_of_all_elements_located_by_partial_link_text(link_text='展开报价'): i.click() except Exception:
return p.text() fl_shop1 = Fieldlist( Field(fieldname=FieldName.SHOP_NAME, css_selector='dl > dt > a'), Field(fieldname=FieldName.SHOP_URL, css_selector='dl > dt > a', attr='href', is_debug=True, is_info=True), ) page_shop_1 = Page( name='千岛湖游记店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector= 'body > div.content.cf > div.main > div.search-content.cf > div > div.result > ul > li' ), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) fl_shop2 = Fieldlist( Field( fieldname=FieldName.SHOP_TAG, css_selector= 'body > div.bgf2f2f2 > div.content.cf > div.ctd_main > div.ctd_main_body > div.ctd_content > div.ctd_content_controls.cf > div', is_focus=True, is_info=True), Field( fieldname=FieldName.SHOP_DETAIL, css_selector=
Field(fieldname=FieldName.SHOP_FEATURE, css_selector='', is_info=True, filter_func=get_shop_feature), Field(fieldname=FieldName.SHOP_RATE, css_selector='', is_info=True, filter_func=get_shop_rate)) fl_shop2 = Fieldlist() page_shop_1 = Page( name='途牛景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector= '#niuren_list > div.contentcontainer.clearfix > div.content_bottom > div.main.fl > div.thelist > ul > li', ), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) page_shop_2 = Page() page_shop_2 = Page( name='途牛景点店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector= ' div.theinfo.ticket.clearfix > a > dl > dt > p > span'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True)
Field(fieldname=FieldName.SHOP_ADDRESS, css_selector='dl > dd > div.tourListLeftListMsg > span:nth-child(1)', is_info=True), Field(fieldname=FieldName.SHOP_PHONE, css_selector='dl > dd > div.tourListLeftListMsg > span:nth-child(2)', is_info=True), Field(fieldname=FieldName.SHOP_AREA, css_selector='dl > dd > div.tourListLeftListMsg > span:nth-child(2)', filter_func=get_shop_area, is_info=True)) page_shop_1 = Page( name='大众点评餐饮店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector= 'body > div.mainLayout.newsMainLayout > div.newsLeftLayout.sceneRightLayout > div' ), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) detail_fl_shop2 = Fieldlist( Field( fieldname=FieldName.SHOP_DES, css_selector= 'body > div.mainLayout.newsMainLayout > div.newsLeftLayout.sceneRightLayout > div.newsDetailContent > div.newsDetailConNote > div', is_info=True), ) detail_shop_2 = Page(name='大众点评获取评论分数和数量页面', fieldlist=detail_fl_shop2, is_save=True)
for i in p('div.htl_info_table > table > tbody').items('tr'): item = (lambda x: x if x else '')(i.text()).split('\n') if len(item) >= 2: around.setdefault(item[0], (lambda x: x[1:] if len(x) >= 2 else [''])(item)) return json.dumps(around, ensure_ascii=False) fl_shop2 = Fieldlist( Field(fieldname=FieldName.SHOP_ROOM_RECOMMEND_ALL,css_selector='#hotelRoomBox', attr='innerHTML', filter_func=get_recommend_all_room_dict, pause_time=1, is_focus=True), Field(fieldname=FieldName.SHOP_ROOM_FAVOURABLE,css_selector='#divDetailMain > div.htl_room_table',attr='innerHTML', filter_func=get_favourable_room, is_focus=True), Field(fieldname=FieldName.SHOP_INTRO, css_selector='#hotel_info_comment > div',attr='innerHTML', filter_func=get_hotel_intro, is_focus=True), Field(fieldname=FieldName.SHOP_PHONE, css_selector='#J_realContact', attr='data-real', regex='^([^<]*).*$', repl=r'\1', is_focus=True), Field(fieldname=FieldName.SHOP_STATISTICS, css_selector='#commentList > div.detail_cmt_box',attr='innerHTML',filter_func=get_shop_statistics, is_focus=True), Field(fieldname=FieldName.SHOP_AROUND_FACILITIES, css_selector='#hotel_info_comment > div', attr='innerHTML',filter_func=get_around_facilities, is_focus=True), ) page_shop_1 = Page(name='携程酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#hotel_list > div.hotel_new_list', item_css_selector='ul.hotel_item'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection)) page_shop_2 = Page(name='携程酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='li.hotel_price_icon > div.action_info > p > a'), mongodb=Mongodb(db=TravelDriver.db,collection=TravelDriver.shop_collection), is_save=True) fl_comment1 = Fieldlist( Field(fieldname=FieldName.COMMENT_USER_NAME, css_selector='div.user_info.J_ctrip_pop > p.name'), Field(fieldname=FieldName.COMMENT_TIME, css_selector='div.comment_main > div.comment_txt > div.comment_bar > p > span', regex=r'[^\d-]*'), Field(fieldname=FieldName.SHOP_NAME, css_selector='#J_htl_info > div.name > h2.cn_n', is_isolated=True), Field(fieldname=FieldName.COMMENT_CONTENT, css_selector='div.comment_main > div.comment_txt > div.J_commentDetail'), Field(fieldname=FieldName.COMMENT_USER_IMG, css_selector='div.user_info.J_ctrip_pop > p.head > span > img', attr='src'), Field(fieldname=FieldName.COMMENT_USER_CHECK_IN, css_selector='div.comment_main > p > span.date'), Field(fieldname=FieldName.COMMENT_USER_ROOM, css_selector='div.comment_main > p > a'), Field(fieldname=FieldName.COMMENT_TYPE, css_selector='div.comment_main > p > span.type'), Field(fieldname=FieldName.COMMENT_SCORE, css_selector='div.comment_main > p > span.score', regex=r'[^\d.]*'), Field(fieldname=FieldName.COMMENT_SCORE_TEXT, css_selector='div.comment_main > p > span.small_c', attr='data-value'), Field(fieldname=FieldName.COMMENT_USER_NUM, css_selector='div.user_info.J_ctrip_pop > p.num'),
css_selector='#booking-wrapper', attr='innerHTML', filter_func=get_shop_ticket, is_focus=True), Field(fieldname=FieldName.SHOP_INFO, css_selector= 'div.main-bd > div.main-wrapper > div.clearfix > div.detail-left', attr='innerHTML', filter_func=get_shop_info, is_focus=True), ) page_shop_1 = Page(name='携程景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector='#searchResultContainer > div', item_css_selector='div'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) page_shop_2 = Page( name='携程景点店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div.search_ticket_title > h2 > a'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) def get_comment_user_name(self, _str): return _str.split(' ')[0]
return _str; fl_shop2 = Fieldlist( #phoenix_dom_3_0 > div > div.head-wrapper.c-title.c-color.c-flexbox.c-line-bottom > div.left > span #phoenix_dom_3_1 > div > div.head-wrapper.c-title.c-color.c-flexbox.c-line-bottom > div.left > span Field(fieldname=FieldName.SHOP_COMMENT_NUM, css_selector='div.card-box.special2-box.c-container >div.head-wrapper.c-title.c-color.c-flexbox.c-line-bottom > div.left > span',is_info=True), Field(fieldname=FieldName.SHOP_SCORE, css_selector='span.left-header-visit',is_info=True), Field(fieldname=FieldName.SHOP_CATEGORY_NAME,css_selector='span.left-header-stdtag',is_info=True), Field(fieldname=FieldName.SHOP_PRICE,css_selector='span.left-header-reference-price',is_info=True), Field(fieldname=FieldName.SHOP_NAME_SEARCH_KEY,css_selector='div.generalHead-left-header-title > span',filter_func=get_shop_name,is_info=True), Field(fieldname=FieldName.SHOP_PHONE,css_selector='#generalinfo > div.generalInfo-address-telnum > div.generalInfo-telnum.item > span.clampword.generalInfo-telnum-text',is_info=True) #generalheader > div.generalHead-left-header.animation-common > div.generalHead-left-header-title > span ) #card-1 > div > ul > li:nth-child(1) > div.cf.mb_5 > div.ml_30.mr_85 > div:nth-child(2) page_shop_1 = Page(name='百度餐饮店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='ul.poilist > li'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection),is_save=True) page_shop_2 = Page(name='百度餐饮店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div.cf > div.ml_30 > div:nth-child(1) > span > a'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection),is_save=True) def get_shop_lng(self,_str): doc = _str.split(',') #再某一个经度范围内 if(float(doc[0]) <= 119.243071 and float(doc[0]) >= 118.650908): return doc[0] else: return 119.051491
from spider.driver.base.mongodb import Mongodb from spider.driver.base.tabsetup import TabSetup fl_weixin1 = Fieldlist( Field(fieldname='public_name', css_selector='div > div.txt-box > p.tit > a', regex=r'[^\u4e00-\u9fa5]*'), ) fl_weixin2 = Fieldlist( Field(fieldname='article_name', css_selector='div > div > h4'), Field(fieldname='article_time', css_selector='div > div > p.weui_media_extra_info'), ) page_weixin_1 = Page(name='微信公众号列表页面', fieldlist=fl_weixin1, listcssselector=ListCssSelector( list_css_selector='#main > div.news-box > ul > li')) page_weixin_2 = Page( name='微信公众号文章列表页面', fieldlist=fl_weixin2, tabsetup=TabSetup(click_css_selector='div > div.txt-box > p.tit > a'), listcssselector=ListCssSelector(list_css_selector='#history > div')) class WeixinSpider(Driver): def __init__(self, isheadless=False, ismobile=False, isvirtualdisplay=False, spider_id='',
css_selector='', filter_func=get_shop_feature, is_info=True), Field(fieldname=FieldName.SHOP_RATE, css_selector='', filter_func=get_shop_rate, is_info=True), Field(fieldname=FieldName.SHOP_COMMENT_URL, css_selector='a', attr='href', filter_func=get_comment_url, is_info=True)) page_shop_1 = Page(name='驴妈妈景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector='#ticket_searchListUl1 > li'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) def get_comment_time(self, _str): return _str[0:10] def get_comment_grade(self, _str): print(_str) width = re.findall(r'[\d]{1,3}', _str)[0] print(float(width) / 150 * 5) return str(float(width) / 150 * 5)
is_info=True), Field(fieldname=FieldName.SHOP_GRADE, css_selector='div > div:nth-child(2) > ul > li:nth-child(1) > a > b', filter_func=get_shop_grade, is_info=True), Field(fieldname=FieldName.SHOP_RATE, css_selector='', filter_func=get_shop_rate, is_info=True), ) fl_shop2 = Fieldlist() page_shop_1 = Page(name='驴妈妈景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector='#mainHotelLeft > div', ), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) page_shop_2 = Page() page_shop_2 = Page(name='驴妈妈景点店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='dl > dt > a'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection)) def get_comment_user_name(self, _str): return _str.split(' ')[0]
from selenium.webdriver.remote.webelement import WebElement from spider.driver.base.tabsetup import TabSetup from spider.driver.base.field import Field, Fieldlist from spider.driver.base.page import Page, PageGroup from spider.driver.base.listcssselector import ListCssSelector from spider.driver.base.mongodb import Mongodb fl = Fieldlist(Field(fieldname=12), Field(fieldname=13)) mongo = Mongodb(db='122', collection='12') lcs = ListCssSelector(list_css_selector=12) tab = TabSetup(url_name=12) p = Page(name=122, fieldlist=fl, mongodb=mongo, listcssselector=lcs, tabsetup=tab) p1 = Page(name=123, fieldlist=fl, mongodb=mongo, listcssselector=lcs, tabsetup=tab) pg = PageGroup(p, p1) print(next(pg))
if len(star) == 2: stars.setdefault(star[0], re.sub(r'[^\d]*', '', star[1])) tag_star_dict.setdefault('star', stars) return json.dumps(tag_star_dict, ensure_ascii=False) fl_shop2 = Fieldlist( Field(fieldname=FieldName.SHOP_GRADE, css_selector='#poi-detail > div.container > div.base-info > div.main-detail.clearfix > div.main-detail-right > div.hotel-appraise > div.hotel-scope > span', pause_time=5, is_focus=True), Field(fieldname=FieldName.SHOP_PHONE, css_selector='#poi-detail > div.container > div.base-info > div.main-detail.clearfix > div.main-detail-left > div.main-detail-left-top.clearfix > div.hotel-detail-info > div > div.call-info > div > span.call-number', is_focus=True), Field(fieldname=FieldName.SHOP_ADDRESS, css_selector='#poi-detail > div.container > div.base-info > div.main-detail.clearfix > div.main-detail-left > div.main-detail-left-top.clearfix > div.hotel-detail-price > div.hotel-address-box.clearfix > span.hotel-address', is_focus=True), Field(fieldname=FieldName.SHOP_ROOM_RECOMMEND_ALL, css_selector='#deal', attr='innerHTML', filter_func=get_shop_room_all, is_focus=True), Field(fieldname=FieldName.SHOP_INTRO, css_selector='#poi-detail > div.container > div.sub-content.clearfix > div.main > div> div.hotel-info', attr='innerHTML', filter_func=get_shop_intro, is_focus=True), Field(fieldname=FieldName.SHOP_STATISTICS, css_selector='#poi-detail > div.container > div.sub-content.clearfix > div.main > div.user-comment-info', attr='innerHTML', filter_func=get_shop_statistics, is_focus=True), Field(fieldname=FieldName.SHOP_COMMENT_NUM, css_selector='#comment > div > h2 > a > span.count', regex=r'[^\d]*'), ) page_shop_1 = Page(name='大众点评酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#poi-list > div.content-wrap > div > div.list-wrapper > div.content > ul > li',item_start=11,item_end=12), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection)) page_shop_2 = Page(name='大众点评酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div.hotel-info-ctn > div.hotel-info-main > h2 > a.hotel-name-link'),mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) def get_rate(self, _str): return str(int(re.sub('[^\d]*','',_str))/10) def get_comment_rate_tag(self, _str): p = PyQuery(_str) tag_list = [] for i in p('span.item').items(): tag_list.append(i.text().strip()) return json.dumps(tag_list, ensure_ascii=False) def get_comment_content(self, _str): return PyQuery(_str).text().replace('收起评论','')
css_selector='', filter_func=_get_shop_comment_num, is_info=True), #无shop_feature Field(fieldname=FieldName.SHOP_FEATURE, css_selector='div:nth-child(2) > div:nth-child(2) > span', is_info=True), Field(fieldname=FieldName.SHOP_RATE, css_selector='div:nth-child(2) > div:nth-child(3) > span', is_info=True)) page_shop_1 = Page(name='飞猪景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector='#tus-recycleview > div > div', item_css_selector='div', item_start=4), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) fl_shop2 = Fieldlist( Field( fieldname=FieldName.SHOP_NAME, css_selector= 'body > div > div.rax-scrollview > div > div:nth-child(1) > div > div:nth-child(1) > span' ), Field( fieldname=FieldName.SHOP_COMMENT_URL, css_selector= 'body > div > div.rax-scrollview > div > div:nth-child(1) > div > div:nth-child(3) > div:nth-child(2)',
Field(fieldname=FieldName.SHOP_FACILITIES, css_selector='#_j_hotel_info', attr='innerHTML', filter_func=get_shop_facilities, offset=6, try_times=10, pause_time=1), Field(fieldname=FieldName.SHOP_STATISTICS, css_selector='#_j_comment', attr='innerHTML', filter_func=get_shop_stattistics), ) page_shop_1 = Page( name='马蜂窝酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#_j_hotel_list > div'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection)) page_shop_2 = Page(name='马蜂窝酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div.hotel-pic > a'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) class MafengwoHotelSpider(TravelDriver): def get_shop_info(self): try: shop_data_list = self.from_page_get_data_list(page=page_shop_1)
filter_func=get_shop_rate), Field(fieldname=FieldName.SHOP_TAG, css_selector='div.txt > span.comment-list', attr='innerHTML', filter_func=get_shop_tag, pause_time=1), Field(fieldname=FieldName.SUBTYPE_NAME, css_selector='div.txt > div.tag-addr > a:nth-child(1)', filter_func=get_shop_subtype_name), Field(fieldname=FieldName.SHOP_ADDRESS, css_selector='div.txt > div.tag-addr > span.addr')) page_shop_1 = Page(name='大众点评爱车店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector( list_css_selector='#shop-all-list > ul > li'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) def get_shop_time(self, _str): try: p = PyQuery(_str) shop_time = '' for i in p('p.info.info-indent').items(): if '营业时间' in i.text(): shop_time = i.text() return shop_time except Exception: return ''
Field( fieldname=FieldName.SHOP_FEATURE, css_selector='div > div.sight_item_about > div > div.intro.color999'), Field( fieldname=FieldName.SHOP_GRADE, css_selector= 'div > div.sight_item_about > div > div.clrfix > div > span.product_star_level > em > span', attr='style', filter_func=get_shop_grade, is_info=True), ) fl_shop2 = Fieldlist() page_shop_1 = Page( name='去哪儿景点店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#search-list > div'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True) page_shop_2 = () page_shop_2 = Page( name='去哪儿景点店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup( click_css_selector='div > div.sight_item_about > h3 > a'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True)