async def _crawl_tb_one_shop_intro_page_info(self, first_shop_title_ele, shop_title) -> dict: """ 抓取店铺介绍页信息 :return: """ get_current_func_info_by_traceback(self=self, logger=self.lg) # 点击shop title ele进店 # self.lg.info('++++++ {} ele exists is {}'.format(shop_title, first_shop_title_ele.exists())) # self.lg.info('即将点击first_shop_title_ele...') first_shop_title_ele.click() # TODO 此处可不睡眠, 因为该元素出现较快, atx会等待 # await async_sleep(3.) # 点击粉丝数进入店铺印象页面 self.d(descriptionMatches='粉丝数\d+.*?', className="android.view.View").click() await async_sleep(6.5) # 无法定位, pass # manager_name = await self._get_manager_name() # 改用读取本地缓存名字 manager_name = await self._read_now_manager_name_file() phone_list = await self._get_shop_phone_num_list() address = await self._get_shop_address() shop_intro_page_info = { 'shop_name': shop_title, 'phone_list': phone_list, 'address': address, 'manager_name': manager_name, } return shop_intro_page_info
async def db_script0( self, select_sql_str: str, update_sql_str: str, func_get_params, ): get_current_func_info_by_traceback(self=self) db_res = self.sql_cli._select_table(sql_str=select_sql_str, ) db_res = [] if db_res is None else db_res if db_res == []: print('目标db_res为空list! 跳过此次!') return None for item in db_res: params = func_get_params(k=item) self.sql_cli._update_table( sql_str=update_sql_str, params=params, ) try: del db_res except: pass return None
async def get_db_res(self) -> list: """ 获取目标goods_id_list :return: """ get_current_func_info_by_traceback(self=self, logger=self.lg) db_res = [] try: self.lg.info('清除过期优惠券ing ...') # 清除过期优惠券 self.sql_cli._delete_table( sql_str= 'delete from dbo.coupon_info where GETDATE()-end_time >= 3', params=None, ) self.lg.info('休眠15s ...') await async_sleep(15) self.lg.info('获取新待检测的goods数据ing...') db_res = list(self.sql_cli._select_table(sql_str=self.sql_tr0, )) except Exception: self.lg.error('遇到错误:', exc_info=True) self.sql_cli = SqlServerMyPageInfoSaveItemPipeline() assert db_res != [] self.lg.info('db_res_len: {}'.format(len(db_res))) return db_res
async def _fck_run(self): print(get_current_func_info_by_traceback(self=self,)) # todo 不删除旧数据, 原因abort 脚本挂了, 能从之前的开始继续!(数据保留) # await self.clear_over_redis_old_tasks() while True: print('\nnow_time: {}'.format(get_shanghai_time())) await self.execute_all_create_dcs_tasks() print('sleep {} s ...'.format(self.sleep_time)) await async_sleep(self.sleep_time)
async def _get_now_page_shop_title_list(self) -> list: """ 获取now_page_shop_title_list :return: """ get_current_func_info_by_traceback(self=self, logger=self.lg) # TODO 不适用instance, 而是用ele index索引的原因是 # instance会导致循环到一定个数出现instance=0对应的某元素不在当前page, 而无法被点击, 导致后续操作紊乱 # 当前页面的shop_title list now_page_shop_title_list = [ item.info.get('text', '') for item in self.d( resourceId="com.taobao.taobao:id/shopTitle", className="android.widget.TextView", clickable=False, ) ] self.lg.info('now_page_shop_title_list: {}'.format( str(now_page_shop_title_list))) assert now_page_shop_title_list != [], 'now_page_shop_title_list不为空list!' return now_page_shop_title_list
async def get_db_res(self) -> list: """ 获取目标goods_id_list :return: """ get_current_func_info_by_traceback(self=self, logger=self.lg) db_res = [] try: # 清除过期优惠券 self.sql_cli._delete_table( sql_str= 'delete from dbo.coupon_info where GETDATE()-end_time >= 3', params=None, ) await async_sleep(15) db_res = list(self.sql_cli._select_table(sql_str=self.sql_tr0, )) except Exception: self.lg.error('遇到错误:', exc_info=True) assert db_res != [] self.lg.info('db_res_len: {}'.format(len(db_res))) return db_res
async def _search_shop_info_by_one_keyword(self, keyword) -> list: """ 根据某关键字进行相应采集 :param keyword: :return: """ get_current_func_info_by_traceback(self=self, logger=self.lg) self.lg.info( '--->>> 即将开始采集 keyword: {} 对应的shop info...'.format(keyword)) # 搜索的是否为店铺 is_shop_search = False # 输入内容 self.d(resourceId="com.taobao.taobao:id/searchEdit").send_keys(keyword) # 搜索 self.d(resourceId="com.taobao.taobao:id/searchbtn").click() await async_sleep(2) if not is_shop_search: # 点击店铺(只需要点击一次, 后续都是以店铺来搜索的) self.d(resourceId="com.taobao.taobao:id/tab_text", text=u"店铺").click() await async_sleep(2) # 点击销量优先 self.d(resourceId="com.taobao.taobao:id/show_text", text=u"销量优先", description=u"销量优先", className="android.widget.TextView").click() is_shop_search = True await async_sleep(2) if self.d( resourceId="com.taobao.taobao:id/tipTitle", text=u"没有搜索结果", className="android.widget.TextView")\ .exists(): # 处理没有搜索结果的 self.lg.info('### 该关键字: {} 无搜索结果!'.format(keyword)) return [] # TODO 方案一无法再ele list里面定位某个特定ele, 并进行后续操作! pass # 方案2: 滑动一个采集一个 self.first_swipe_height, self.second_swipe_height = await self._get_first_swipe_height_and_second_swipe_height( ) self.lg.info( 'self.first_swipe_height: {}, self.second_swipe_height: {}'.format( self.first_swipe_height, self.second_swipe_height)) # 先上滑隐藏全部, 天猫, 店铺, 淘宝经验 await u2_up_swipe_some_height(d=self.d, swipe_height=self.first_swipe_height) shop_name_list = [] shop_crawl_count = 1 res = [] while shop_crawl_count < self.max_shop_crawl_count: if self.d(resourceId="com.taobao.taobao:id/loading_text", text=u"没有更多内容了", className="android.widget.TextView").exists(): self.lg.info('该关键字下滑,无更多内容!! 跳过该关键字采集!') break try: now_page_shop_title_list = await self._get_now_page_shop_title_list( ) except (AssertionError, UiObjectNotFoundError): self.lg.error('遇到错误:', exc_info=True) # 从店铺首页返回搜索页 await u2_page_back(d=self.d, back_num=1) await u2_up_swipe_some_height( d=self.d, swipe_height=self.second_swipe_height / 2) # 等待新返回的list成功显示 await async_sleep(1.) continue try: # clickable = False 确定当前页面某btn是否已被点击, 未被点击 False first_shop_title_ele = self.d( resourceId="com.taobao.taobao:id/shopTitle", className="android.widget.TextView", # instance=0, text=now_page_shop_title_list[1] if shop_crawl_count != 1 else now_page_shop_title_list[0], # 保证每个都被遍历 clickable=False, ) except IndexError: self.lg.error('遇到错误:', exc_info=True) # 无法定位first_shop_title_ele元素时 await u2_up_swipe_some_height( d=self.d, swipe_height=self.second_swipe_height / 2) # 等待新返回的list成功显示 await async_sleep(1.) continue try: shop_title = first_shop_title_ele.info.get('text', '') except (UiObjectNotFoundError, ): continue if await self._shop_title_has_been_traversed( shop_title=shop_title, shop_name_list=shop_name_list, ): continue self.lg.info( '正在采集店名: {}, shop_crawl_count: {}, keyword: {} ...'.format( shop_title, shop_crawl_count, keyword)) shop_name_list.append(shop_title) try: shop_intro_page_info = await self._crawl_tb_one_shop_intro_page_info( first_shop_title_ele=first_shop_title_ele, shop_title=shop_title, ) send_res = await self._send_2_tb_shop_info_handle( one_dict=shop_intro_page_info) res.append(shop_intro_page_info) if send_res == 'success'\ and shop_title not in self.company_name_list_filter: # 存入成功储存到db中的 self.company_name_list_filter.add(shop_title) except (UiObjectNotFoundError, AssertionError): self.lg.error('遇到错误:', exc_info=True) await self._back_2_search_page( first_shop_title_ele=first_shop_title_ele) if shop_title not in self.company_name_list_filter: # 存入异常退出但是已被遍历的, 即不管结果如何, 已遍历的都进行存储!! self.company_name_list_filter.add(shop_title) continue # 返回搜索结果list页面 # 默认返回前两页会页面混乱 # await u2_page_back(d=self.d, back_num=2) # 改用元素定位来看是否成功返回上层 await self._back_2_search_page( first_shop_title_ele=first_shop_title_ele) await u2_up_swipe_some_height( d=self.d, swipe_height=self.second_swipe_height) # await async_sleep(2) # 等待新返回的list成功显示 shop_crawl_count += 1 self.lg.info('--->>> 采集 keyword: {} 对应的shop info 完毕!'.format(keyword)) return res