def updateGeohash_taskbase(): sql = ''' select Lng,Lat,TaskId from taskbase where IsExcute = 4 ''' _base32 = '0123456789bcdefghjkmnpqrstuvwxyz' _encode_map = {} for i in range(len(_base32)): _encode_map[i]=_base32[i] DbContext = DbHelper() result = DbContext.Query(sql) for item in result: Lng = item['Lng'] Lat = item['Lat'] TaskId = item['TaskId'] geohash = getGeoHashByLngLat(float(Lat),float(Lng),_encode_map) sql = ''' update taskbase set GenHash = '%s' where TaskId = %d ''' sql = sql % (geohash,TaskId) DbContext.Update(sql) print(str(TaskId) + '更新完') pass
def updateGeohash(): sql = ''' select Lng,Lat,shopName,address from shop where address = '上海市浦东新区万祥镇严木桥路96号1-2层' ''' _base32 = '0123456789bcdefghjkmnpqrstuvwxyz' _encode_map = {} for i in range(len(_base32)): _encode_map[i]=_base32[i] DbContext = DbHelper() result = DbContext.Query(sql) for item in result: Lng = item['Lng'] Lat = item['Lat'] shopname = item['shopName'] address = item['address'] geohash = getGeoHashByLngLat(float(Lat),float(Lng),_encode_map) sql = ''' update shop set AddressGeohash = '%s' where shopName = '%s' and address = '%s' ''' sql = sql % (geohash,shopname,address) DbContext.Update(sql) pass
def SearchCatchStore(storeName: str, poco, device, DeviceNum: str, DeviceType: int): # 查找到该门店的相关信息 AllClassifyInputClickNum = 0 # 大类进入点击小类的次数 IsAllClassifyInput = False # 是否从大类搜索进入(默认不是) DbContext = DbHelper() # result = DbContext.GetStorePoint(storeId) result = DbContext.GetStorePointForName(storeName) if len(result) == 1: # address字段可能找不到这个店,用AnchorPoint才能找到这个店 strs = str(result[0]['AnchorPoint']).split(';') StoreAddress = strs[0] StoreName = result[0]['shopName'] StoreCity = result[0]['City'] storeId = result[0]['mtWmPoiId'] IsAddress = SwithPosition(poco, StoreAddress, StoreCity, storeName, 0) # 切换定位 isExists = getStore(IsAddress, poco, DeviceNum, IsAllClassifyInput, StoreName, device) if(not isExists): BackHomePage(poco, DbContext, DeviceNum, device) # 返回首页 StoreAddress = result[0]['address'] IsAddress = SwithPosition(poco, StoreAddress, StoreCity, storeName, 0) # 切换定位 isExists = getStore(IsAddress, poco, DeviceNum, IsAllClassifyInput, StoreName, device) if(not isExists): # DbContext.AddLog(DeviceNum, 2, '爬取['+StoreName+']失败,没找到店铺') print('该店铺不存在!') pass
def BatchSendMessage(self): sql = ''' update task set IsExcute = 1 where IsExcute = 2 ''' DbContext = DbHelper() DbContext.Update(sql) pass
def delStore(self): DbContext = DbHelper() storelist = open("log.txt", mode='r') for index in range(16901): line = next(storelist) delline = str(line).split(',')[0] relust = DbContext.deleteStore(delline) print("删除第%d行 - %s" % (index, delline) + str(relust)) pass
def getAddressFromGaoDe(): DbContext = DbHelper() sql = """ select Lng,Lat from address where city like '%重庆%' and Lng is not null and Lat is not null and RepresentativeAdress is null limit 5500 """ allLngLat = DbContext.Query(sql) key = '929d86bb3c93b6895554459ab7893171' print(str(len(allLngLat))) for item in allLngLat: try: #print(str(float(item["Lng"])),str(float(item["Lat"]))) #url = url % (key,float(),float(item["Lat"])) url = 'https://restapi.amap.com/v3/geocode/regeo?key=' + key + '&location=' + str(item["Lng"]) + ',' + str(item["Lat"]) + '&poitype=&radius=1000&extensions=base&batch=false&roadlevel=1' req = rq.get(url) dataDic = json.loads(req.text) address = '' businessaddress = '' priority = 3 if int(dataDic['status']) == 1: baseaddress = str(dataDic['regeocode']['formatted_address']) #基础地址 if 'addressComponent' in dataDic['regeocode']: if 'businessAreas' in dataDic['regeocode']['addressComponent']: if len(dataDic['regeocode']['addressComponent']['businessAreas']) > 0: result = dataDic['regeocode']['addressComponent']['businessAreas'][0] print(dataDic['regeocode']['addressComponent']['businessAreas']) if len(result) > 0: if 'name' in result: print(222) businessaddress = ' ' + result["name"] priority = 1 address = baseaddress + businessaddress else: address = '' continue if address != '' and address != '[]': address = address.replace("'","") UpdateAddress(float(item["Lng"]),float(item["Lat"]),str(address),priority,DbContext) print(address) except Exception as e: if 'not all arguments converted during string formatting' in repr(e): continue else: print('出错:'+ repr(e)) break pass pass
def __GoHeavy(self, taskid, execid, sendMessageList): try: r = CacheData() for msg in sendMessageList: if not r.CurrentlyData(taskid, execid, msg['Id']): #去掉重复的 sendMessageList.remove(msg) #判断是否要清除缓存 r.ActiveClear(taskid, execid) except Exception as e: DbContext = DbHelper() DbContext.AddLog( '', 4, 'redis去重异常异常:' + repr(e).replace("'", "").replace("\"", "")) del DbContext return sendMessageList
def sendErrorMessage(self, message): try: #创建通道 channel = self._conn.channel() # channel.queue_declare(queue=self.__QueueName) channel.basic_publish(exchange='', routing_key=self.__QueueName, body=message, properties=pika.BasicProperties( delivery_mode=2, )) except Exception as e: DbContext = DbHelper() DbContext.AddLog( '', 4, '发送队列出错数据异常:' + repr(e).replace("'", "").replace("\"", "")) print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) print('发送队列出错数据异常:' + repr(e).replace("'", "").replace("\"", "")) del DbContext
def ActiveClear(self, taskid, execid): try: sql = ''' select count(*) as num from task where TaskTag = '%s' and Exec = '%s' and IsExcute < 3 ''' % (str(taskid), str(execid)) DbContext = DbHelper() result = DbContext.Query(sql) #等于1说明当前只剩下这个任务没有回传,因此可以清除 if result[0]['num'] == 1: #清除这个set SetName = str(taskid) + str(execid) members = self.__conn.smembers(SetName) for mem in members: self.__conn.srem(SetName, mem) del DbContext except Exception as e: print(repr(e)) pass
def sendMessage(self, taskId, messageList): try: message = self.__DealSendMessage(taskId, messageList) #创建通道 channel = self._conn.channel() #channel.queue_declare(queue=self.__QueueName,durable=True) EncodeTextMsg = EncodeText(message) channel.basic_publish(exchange='', routing_key=self.__QueueName, body=EncodeTextMsg, properties=pika.BasicProperties( delivery_mode=2, )) except Exception as e: DbContext = DbHelper() DbContext.AddLog( '', 4, '发送数据异常:' + repr(e).replace("'", "").replace("\"", "")) print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) print('发送队列出错数据异常:' + repr(e).replace("'", "").replace("\"", "")) del DbContext return False else: return True
def __DealSendMessage(self, taskId, sendMessageList): Result = str(sendMessageList) Data = {} Data['State'] = 1 DbContext = DbHelper() sql = ''' select Id,StoreId,Lng,Lat,GeoHash,Province,CityCode,District,TaskTag,Exec from task where TaskId = %d ''' % (int(taskId)) receiveMsgList = DbContext.Query(sql) receiveMsgDict = receiveMsgList[0] Data['Id'] = receiveMsgDict['Id'] Data['StoreId'] = receiveMsgDict['StoreId'] Data['Lng'] = receiveMsgDict['Lng'] Data['Lat'] = receiveMsgDict['Lat'] Data['GeoHash'] = receiveMsgDict['GeoHash'] Data['Platform'] = 'mt' Data['Province'] = receiveMsgDict['Province'] Data['City'] = receiveMsgDict['CityCode'] Data['District'] = receiveMsgDict['District'] Data['Task'] = receiveMsgDict['TaskTag'] Data['Exec'] = receiveMsgDict['Exec'] Data['Type'] = 'StoreGet' if len(sendMessageList) > 0: Data['Exists'] = 1 sendMessageList = self.__GoHeavy(Data['Task'], Data['Exec'], sendMessageList) if len(sendMessageList) > 0: Data['Exists'] = 1 else: Data['Exists'] = 0 Result = str(sendMessageList) else: Data['Exists'] = 0 Data['Result'] = Result return str(Data)
def getLngLatByAddress(): key = '929d86bb3c93b6895554459ab7893171' sql = ''' select shopName,city,address from shop s where s.Genhash is null ''' try: DbContext = DbHelper() Alladdress = DbContext.Query(sql) for address in Alladdress: _shopname = address['shopName'] _city = address['city'] _address = address['address'] url = 'https://restapi.amap.com/v3/geocode/geo?key='+ key +'&address=' + _address +'&city=' + _city req = rq.get(url) dataDic = json.loads(req.text) if int(dataDic['status']) == 1: geocodes = dataDic['geocodes'] if len(geocodes) > 0: location = geocodes[0]['location'].split(",") if len(location) > 0: Lng = location[0] Lat = location[1] sql = ''' update shop set Lng = '%s', Lat = '%s' where shopName = '%s' and address = '%s' ''' sql = sql % (str(Lng),str(Lat),_shopname,_address) DbContext.Update(sql) print('已更新【'+ _shopname +'】') pass except Exception as e: print(repr(e)) pass
def getLngLatByAddress_task(): key = '929d86bb3c93b6895554459ab7893171' sql = ''' select TaskId,Address from taskbase t where t.IsExcute = 4 ''' try: DbContext = DbHelper() Alladdress = DbContext.Query(sql) for address in Alladdress: _TaskId = address['TaskId'] _address = address['Address'] url = 'https://restapi.amap.com/v3/geocode/geo?key='+ key +'&address=' + _address +'&city=福州' req = rq.get(url) dataDic = json.loads(req.text) if int(dataDic['status']) == 1: geocodes = dataDic['geocodes'] if len(geocodes) > 0: location = geocodes[0]['location'].split(",") if len(location) > 0: Lng = location[0] Lat = location[1] sql = ''' update taskbase set Lng = '%s', Lat = '%s' where TaskId = %d ''' sql = sql % (str(Lng),str(Lat),_TaskId) DbContext.Update(sql) print('已更新【'+ _address +'】') pass except Exception as e: print(repr(e)) pass
def RunningCheck(): print( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 设备运行情况检测 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' ) RunDevice = [] # 检查设备程序是否在运行 for proc in psutil.process_iter(): try: pinfo = proc.as_dict(attrs=['cmdline']) if type(pinfo['cmdline']).__name__ != 'NoneType': if len(pinfo['cmdline']) > 0: # 格式['python','main.py'] if '/bin/sh' == pinfo['cmdline'][0] and pinfo['cmdline'][ 1] == '-c': command = str(pinfo['cmdline'][2]) print(command) command = command.replace(".py", "").replace( "PyDaemon", "").replace('./', "").replace(" ", "").replace("-d", "") if 'python3.7' in command: deviceNum = command.replace("python3.7", "")[:16] if deviceNum not in RunDevice: RunDevice.append(deviceNum) pass except psutil.NoSuchProcess as e: print( '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 检测设备状态异常 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' ) print(repr(e)) continue # 这里会限制只能重启8台 if len(RunDevice) < 9: # 获取没有运行的设备 DbContext = DbHelper() DeviceDict = [ '5LM0216902001108', '5LM0216910000994', '5LM0216B03001264', 'APU0216408028484', 'DLQ0216630004610', 'E4J4C17405011422', 'DLQ0216729004546' ] DeviceList = [] for devicenum in DeviceDict: DeviceList.append(devicenum) NotRunningDevice = list(set(DeviceList).difference(set(RunDevice))) if len(NotRunningDevice) > 0: # 启动没有运行的设备的Daemon for deviceNum in NotRunningDevice: try: os.popen('python3.7 ' + deviceNum + '.py >> /root/airtest/log/Device/Runniglog' + datetime.datetime.now().strftime("%Y%m%d") + '.log') DbContext.AddLog(deviceNum, 1, '启动设备[' + deviceNum + ']执行任务') except Exception as e: print('\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 重启设备[' + deviceNum + ']异常 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') DbContext.AddLog( deviceNum, 3, '重启设备[' + deviceNum + ']异常 ' + repr(e).replace("'", "")) pass
def StartCapture(poco, AllPosition, DeviceType, TargetCity, DeviceNum, cityCode, device): CloseUpDateInfo(poco) IsAllClassifyInput = False # 是否从大类搜索进入(默认不是) AllClassifyInputClickNum = 0 # 大类进入点击小类的次数 currentTaskResult = [] # 存放本次任务抓取的门店信息 if poco(text="美团外卖").exists(): poco(text="美团外卖").click() if DeviceNum == 'E4J4C17405011422': poco.swipe([0.5, 0.5], [0.5, 0.6], duration=0.3) IsAddress = SwithPosition(poco, AllPosition[0]['RepresentativeAdress'], TargetCity, DeviceNum, 0) # 第一次切换定位 if not IsAddress: return currentTaskResult, False print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) try: if poco(text="送药上门").exists(): poco(text="送药上门").wait(waitTime).click([0, 0]) else: if DeviceNum == 'E4J4C17405011422': poco.swipe([0.5, 0.6], [0.5, 0.5], duration=0.3) result = SearchDrugPage(poco, device, DeviceNum) if not result: print(DeviceNum) if poco(text="美食").exists(): IsAddress = False print( '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 第一个定位点无送药上门 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' ) else: print( '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 被反爬检测到了 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' ) return currentTaskResult, True else: IsAllClassifyInput = True except PocoNoSuchNodeException: print(DeviceNum) print( '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 第一个定位点无送药上门f o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' ) IsAddress = False pass StoreList = [] #只存储当前定位的门店名称 SwitchNum = 0 #定位切换次数 DbContext = DbHelper() for addressDic in AllPosition: #结构改了以后AllPosition只会有一条数据 address = addressDic['RepresentativeAdress'] #标志性坐标点 addressGenhash = addressDic['Genhash'] #地理散列值 print('===========================【' + DeviceNum + '】抓取【' + address + '】开始===========================') # 第一次进来不需要切换定位 if SwitchNum > 0: StoreList.clear() #切换一次地址就将列表清空 IsAddress = SwithPosition(poco, address, TargetCity, SwitchNum) #切换定位 if not IsAddress: print('===========================【' + DeviceNum + '】抓取【' + address + '】完成 无店铺===========================\n') continue SongYaoNum = 0 while True: if poco(text="送药上门").exists(): poco(text="送药上门").wait(waitTime).click([0, 0]) break SongYaoNum += 1 time.sleep(1) if SongYaoNum >= 3: print( '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 该定位点无送药上门 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' ) IsAddress = False break SwitchNum += 1 if not IsAddress: print('===========================【' + DeviceNum + '】抓取【' + address + '】完成 无店铺===========================\n') continue storelenNum = 0 swipeNume = 0 # 第一次大幅度滑动 if DeviceType == 1: poco.swipe([0.2, 0.9], [0.2, 0.45], duration=0.3) else: poco.swipe([0.4, 0.9], [0.4, 0.75], duration=0.3) if IsAllClassifyInput: if poco(text="常用药品").exists(): poco(text="常用药品").click() AllClassifyInputClickNum += 1 # 按照销量排序 if poco(text='销量').exists(): poco(text='销量').wait(waitTime).click() # 当前循环是一个定位点的门店 backPage = poco("com.sankuai.meituan.takeoutnew:id/iv_back") bottomElement = poco("com.sankuai.meituan.takeoutnew:id/noMoreView") while True: # 判断程序是否被紧急置停 mode = DbContext.GetDeviceRunningMode(DeviceNum) if mode == 5: device.keyevent("4") return currentTaskResult, True AllStore = poco( "com.sankuai.meituan.takeoutnew:id/fl_fragment_container" ).offspring( "com.sankuai.meituan.takeoutnew:id/pull_to_refresh_view" ).offspring( "com.sankuai.meituan.takeoutnew:id/viewpager_content" ).offspring( "com.sankuai.meituan.takeoutnew:id/wm_st_poi_channel_list" ).child("android.widget.FrameLayout").wait(waitTime) i = 0 if len(AllStore) > 0: # print('当前列表展示的门店数量:'+ str(len(AllStore))) circleIsFail = False for store in AllStore: if i == 0: i += 1 #第一个是顶部筛选 continue storeInfo = {} try: storeNameResult = GetStoreName(store) if not storeNameResult[0]: continue storeName = storeNameResult[2] # 销量、评分 Sell = '0' Score = '评分未知' # time_ScoreUI = storeNameResult[1].child("android.widget.LinearLayout") # try: # time = time_ScoreUI.offspring("com.sankuai.meituan.takeoutnew:id/txt_mt_delivery_time_info").exists() # #如果配送时间不存在就是快递店就不爬 # if not time: # print("-------------快递店不爬【"+storeName+"】---------------") # continue # except PocoNoSuchNodeException: # print("-------------獲取配送時間異常---------------") storeSellResult = GetStoreSell(storeNameResult[1]) if not storeSellResult[0]: continue else: Sell = storeSellResult[2] storeScoreResult = GetStoreScore( storeSellResult[1]) if not storeScoreResult[0]: continue else: Score = storeScoreResult[1] # 该定位点存在且门店名称也存在,有且只有一个地址,那么就不需要继续爬门店地址,更新销量和评分数据即可 IsClickStore = False if storeSellResult[0] and storeScoreResult[ 0]: #销量和评分全部获取到在判断 # 同一定位地址,同店铺名字的不予考虑 if storeName in StoreList: continue else: if '成人用品' in storeName or '情趣' in storeName: #成人用品店不抓取,干扰太大 continue StoreList.append(storeName) storeInfo['Name'] = storeName if '暂无评分' in str(Score): storeInfo['Score'] = 0 else: storeInfo['Score'] = float(Score) Sell_Str = str(Sell).replace("月售", "").replace( "+", "").replace("件", "") storeInfo['Sales'] = float(Sell_Str) storeInfo['Phone'] = '' storeInfo['Brand'] = '' storeInfo['Created'] = datetime.datetime.now( ).strftime("%Y-%m-%d %H:%M:%S") IsClickStore, addressinfo, mtWmPoiId = DbContext.GetStoreInfo( storeName, str(Score), str(Sell).replace("月售", "").replace("+", "").replace( "件", ""), addressGenhash, TargetCity, address) # 继续爬取门店地址 if IsClickStore: # 根据店名+城市找地址 storeAddress, mtWmPoiId = DbContext.UpdateGeoHash( storeName, cityCode, addressGenhash) storeInfo['OriginAddress'] = storeAddress storeInfo['Id'] = mtWmPoiId if FaileAddress in storeAddress: # 根据店名+城市获取地址失败在继续单击查找 store.click() storeAddress = GetStoreInfo( poco, DbContext, DeviceNum) Sell = str(Sell).replace("月售", "").replace( "+", "").replace("件", "") storeAddress = storeAddress.replace( "地址", "").replace("地址:", "") if FaileAddress not in storeAddress: mtWmPoiId = DbContext.InsertShop( storeName, Score, storeAddress, Sell, address, addressGenhash, TargetCity, cityCode) storeInfo['Id'] = mtWmPoiId storeInfo['OriginAddress'] = storeAddress else: storeAddress = '不需要爬取地址' storeInfo['OriginAddress'] = addressinfo storeInfo['Id'] = mtWmPoiId print(storeName + ' 【' + str(Sell) + '】 【评分】:' + str(Score) + ' 【地址】:' + storeAddress + '\n') if FaileAddress not in storeInfo[ 'OriginAddress'] and len(storeInfo['Id']) != 0: storeInfo['OriginAddress'] = str( storeInfo['OriginAddress']).replace( u'\xa0', u' ') currentTaskResult.append(storeInfo) ''' 药品信息 CatchProductResult = ProductInfo.GetProduct(poco) if CatchProductResult == 'error': circleIsFail = True break ''' except PocoNoSuchNodeException: continue except Exception as e: DbContext.AddLog( DeviceNum, 3, '设备[' + DeviceNum + ']爬取门店列表异常:' + repr(e).replace("'", "").replace("\"", "")) continue pass if circleIsFail: print('获取门店列表失败!') break # 判断是否滑动到底部 Isbottom = bottomElement.exists() if Isbottom: print('~~~~~~~~~~~~~~~本次定位的门店到底了~~~~~~~~~~~~~~~') if not ClickClassify(poco, AllClassifyInputClickNum): if backPage.exists(): backPage.wait(waitTime).click() else: device.keyevent("4") print('【因为抓取门店到底而结束本次定位查询】') if IsAllClassifyInput: device.keyevent("4") break else: AllClassifyInputClickNum += 1 continue else: # 当前页面展示的门店数量 if len(AllStore) > 4: if DeviceType == 1: poco.swipe([0.5, 0.8], [0.5, 0.35], duration=0.3) else: poco.swipe([0.5, 0.8], [0.5, 0.5], duration=0.3) else: # 手机展示不会超过4 + 1个 P20展示5+1个 if DeviceType == 1: poco.swipe([0.5, 0.8], [0.5, 0.45], duration=0.3) else: poco.swipe([0.5, 0.8], [0.5, 0.65], duration=0.3) swipeNume += 1 # 滑动了40次(至少是120家店) 且门店数量还是小于一页,那么认为这个也没滑动到底且没有门店滑动到底的提示 if swipeNume > 40: if len(StoreList) < 6 or len(AllStore) < 4: if not ClickClassify(poco, AllClassifyInputClickNum): if backPage.exists(): backPage.wait(waitTime).click() else: device.keyevent("4") # 如果是大类进来的,还要在返回一次 if IsAllClassifyInput: device.keyevent("4") print('认为这个也没滑动到底且没有门店滑动到底的提示而退出') break else: AllClassifyInputClickNum += 1 continue else: # 判断定位点附近是否有门店 if poco(text='该定位下暂无服务商家,请切换地址').exists(): print('~~~~~~~~~~~~~~~~该定位下暂无服务商家,请切换地址~~~~~~~~~~~~~~~~') if not ClickClassify(poco, AllClassifyInputClickNum): if backPage.exists(): backPage.wait(waitTime).click() else: device.keyevent("4") if IsAllClassifyInput: device.keyevent("4") break else: AllClassifyInputClickNum += 1 continue # 跳到登录界面需要退出 if poco(text='获取短信验证码').exists(): device.keyevent("4") continue storelenNum += 1 if storelenNum > 5: print('【因为获取门店列表长度为0而结束本次定位查询】') if not ClickClassify(poco, AllClassifyInputClickNum): if backPage.exists(): backPage.wait(waitTime).click() else: device.keyevent("4") if IsAllClassifyInput: device.keyevent("4") break else: AllClassifyInputClickNum += 1 continue else: if DeviceType == 1: poco.swipe([0.5, 0.9], [0.5, 0.7], duration=0.3) continue pass print('===========================【' + DeviceNum + '】抓取【' + address + '】完成===========================\n') return currentTaskResult, False
def main(DeviceNum): # 多设备连接时,可指定设备编号 # python -m airtest run main.py --device Android://127.0.0.1:5037/0123456789ABCDEF # device = Android('GWY0216C16002906') # 设备类型 1 手机[720,1280][1080,1920] 2平板[1200,1920] # DeviceNum = 'c5bac654' NeedSwipe = ['CLB0218414001154', 'DLQ0216824000142', 'E4J4C17405011422', 'APU0216530000778', 'APU0216408028484', 'APU0216111008105'] try: DbContext = DbHelper() DeviceType = 0 device = Android(DeviceNum) # device.adb.start_shell("su") # device.adb.start_shell("wipe data") # device.adb.start_shell("wipe cache") # device.adb.start_cmd("adb reboot") if '0123456789ABCDEF' not in DeviceNum: device.wake() # 唤醒页面 poco = AndroidUiautomationPoco(device) # if DeviceNum in NeedSwipe: poco.swipe([0.4, 0.9], [0.4, 0.55], duration=0.1) time.sleep(2) # 复位一下,防止之前没有睡眠也滑动 poco.swipe([0.4, 0.45], [0.4, 0.9], duration=0.1) else: poco = AndroidUiautomationPoco( use_airtest_input=True, screenshot_each_action=False) # ClearMemory(device,poco,'') # return device_screen = poco.get_screen_size() device_x = device_screen[0] device_y = device_screen[1] print(str(device_y), str(device_x)) if device_y > 1600 and device_x > 1080: DeviceType = 2 else: DeviceType = 1 if DeviceType == 0: print( '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 设备型号无法确定 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') DbContext.AddLog(DeviceNum, 3, '设备[' + DeviceNum + ']型号无法确定') return elif DeviceType == 1: print( '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 设备型号为手机 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n') else: print( '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 设备型号为平板 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n') except Exception as e: print('\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 设备连接异常 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') DbContext.AddLog( DeviceNum, 3, "设备[" + DeviceNum + "]连接异常:" + repr(e).replace("'", "")) else: BackHomeStatus = BackHomePage( poco, DbContext, DeviceNum, device) # 返回首页 if BackHomeStatus: mode = DbContext.GetDeviceRunningMode(DeviceNum) taskList = [] # 获取当前设备要抓取的城市 IsRunning = True if mode == 1: taskList = DbContext.GetDeviceTask(DeviceNum) elif mode == 2: taskList = DbContext.GetDeviceTaskByMode2() elif mode == 3: taskList = DbContext.GetDeviceTaskByMode3() elif mode == 5: IsRunning = False elif mode == 6: # 根据店名爬取 while True: data = get_shopName() id_name_addr_city = data[0] shopid = id_name_addr_city[0] storeName = id_name_addr_city[1] addr = id_name_addr_city[2] city = (id_name_addr_city[3])[:-1] print(storeName, addr, city) if data[1] == 3: crawl_status_code(storeName, 3) else: crawl_status_code(storeName, 1) search_store(storeName, poco, device, addr, city, shopid) BackHomePage(poco, DbContext, DeviceNum, device) crawl_status_code(storeName, 2) if(len(taskList) == 1): # 更新任务为运行中 DbContext.UpdateTaskStatus( int(taskList[0]['TaskId']), 1, 0, mode) while IsRunning: BackHomeStatus = BackHomePage( poco, DbContext, DeviceNum, device) # 返回首页 if BackHomeStatus: for task in taskList: # AddressList(task['TargetCity']) #获取未抓取的坐标点 AllPosition = [ {'RepresentativeAdress': task['RepresentativeAdress'], 'Genhash':task['Genhash']}] taskId = task['TaskId'] cityCode = task['CityCode'] # 更新任务为运行中 DbContext.UpdateTaskStatus(int(taskId), 1, 0, mode) # 计时 StartTime = datetime.datetime.now() # 返回值需要写进队列 currentTaskResult, IsEmergencyStop = StartCapture( poco, AllPosition, DeviceType, task['TargetCity'], DeviceNum, cityCode, device) # 抓取数据 # 没有紧急置停的情况下才完成后续的更新 if not IsEmergencyStop: EndTime = datetime.datetime.now() DbContext = DbHelper() DbContext.AddLog(DeviceNum, 2, '设备[' + DeviceNum + ']本次抓取[' + AllPosition[0]['RepresentativeAdress'] + '] [' + str( len(currentTaskResult)) + '] 家店,耗时:' + str(((EndTime - StartTime).seconds)/60)) # 更新任务为完成 DbContext.UpdateTaskStatus( int(taskId), 2, len(currentTaskResult), mode) # 将任务的执行结果回写到队列 if mode != 3: Produce = SendMessage() result = Produce.sendMessage( taskId, currentTaskResult) if result: # 将任务状态改为已回写队列 DbContext.UpdateTaskStatus( int(taskId), 3, len(currentTaskResult), mode) else: break mode = DbContext.GetDeviceRunningMode(DeviceNum) if len(taskList) > 0: taskList.clear() if mode == 1: taskList = DbContext.GetDeviceTask(DeviceNum) elif mode == 2: taskList = DbContext.GetDeviceTaskByMode2() elif mode == 3: taskList = DbContext.GetDeviceTaskByMode3() elif mode == 5: print( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 紧急置停 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') break if len(taskList) > 0: print( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 抽取一条任务 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') else: print( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 任务执行完毕 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') break else: DbContext.AddLog( DeviceNum, 3, '设备[' + DeviceNum + ']返回首页异常') break else: DbContext.AddLog(DeviceNum, 3, '设备[' + DeviceNum + ']返回首页异常')
def GetProduct(poco, device, storeId): if (poco("com.sankuai.meituan.takeoutnew:id/category_recycler_view"). exists()): poco("com.sankuai.meituan.takeoutnew:id/category_recycler_view").swipe( [-0.8, 0], duration=0.5) time.sleep(3) #睡眠一段时间等待页面加载 ContainAll = poco(text="全部分类") IsContainAll = ContainAll.exists() AllProductInfoList = [] ErrorMessage = 'success' DbContext = DbHelper() if IsContainAll: #找到父节点 parent = ContainAll.parent() parent.wait(waitTime).click() #全部分类界面 StartTime = datetime.datetime.now() time.sleep(1) #睡眠一段时间等待页面加载 SlideNum = 0 #滑动次数 BottomNum = 0 #到达底部次数 CategaryNum = 0 #爬取的大类数目(一般药店第一个是折扣商品会包含折扣、原价信息,之后的几个大类下的商品则不会包含) #用于存储抓取的商品数量 CurrentProductNum = 0 maxNum = 0 while True: try: # if SlideNum % 5 == 0:#滑动几次之后大概睡眠一会 # time.sleep(1) #睡眠一段时间等待页面加载 # else: #poco.swipe([0.5,0.7],[0.5,0.4],duration = 0.3) #判断是否出现释放展示下一类按钮 # nextCategary = poco("com.sankuai.meituan.takeoutnew:id/tv_refresh").wait(waitTime).exists() # if nextCategary: # print('===================当前药品大类抓取完成===================') # CategaryNum += 1 # pass # SlideNum += 1 #当前要抓取的大类所能看见的界面展示 ''' if nextCategary: time.sleep(1) nextCategary = False''' time.sleep(1) CurrentClassfiy = poco( "com.sankuai.meituan.takeoutnew:id/layout_shop_root_scroll_container" ).offspring( "com.sankuai.meituan.takeoutnew:id/priority_scrollview" ).offspring("com.sankuai.meituan.takeoutnew:id/poi_pinned_layout" ).child("android.support.v7.widget.RecyclerView" ).child("android.widget.FrameLayout") if (len(CurrentClassfiy) == 0): CurrentClassfiy = poco( "com.sankuai.meituan.takeoutnew:id/recycler" ).offspring( "com.sankuai.meituan.takeoutnew:id/ll_stickyfoodList_adapter_food_food" ) if (len(CurrentClassfiy) == 0): CurrentClassfiy = poco( "com.sankuai.meituan.takeoutnew:id/recycler" ).offspring( "com.sankuai.meituan.takeoutnew:id/ll_stickyfoodList_adapter_food_food" ) if len(CurrentClassfiy) > 0: message = '【名称】:%s 【售价】:%s 【原价】:%s 【销量】:%s 【折扣】:%s' print("本批抓到" + str(len(CurrentClassfiy)) + "个") for product in CurrentClassfiy: try: #商品信息 # if(product.offspring(name = 'com.sankuai.meituan.takeoutnew:id/ll_stickyfoodList_adapter_food_food').exists()): # product =product.offspring(name = 'com.sankuai.meituan.takeoutnew:id/ll_stickyfoodList_adapter_food_food') # if len(product)== 0: # print("当前为非商品标签") # continue #名称 _productName = GetProductName(product, poco) if _productName == '获取名称失败': continue if _productName in AllProductInfoList: print("【" + _productName + "】商品已存在") continue #之前爬取过的就不做处理 else: # _productPrice = product.child(name = 'com.sankuai.meituan.takeoutnew:id/ll_stickysold_count_unit_price_original_price_fix').child(name = 'com.sankuai.meituan.takeoutnew:id/ll_price_layout') #销量 _productSalesVolume = GetSale(product, poco) #售价 _productCurrentPrice = GetPrice(product, poco) if _productCurrentPrice != -1: #只有拿到售价的才算爬到商品的信息 AllProductInfoList.append(_productName) #原价 if CategaryNum < 2: _productBeforePrice = GetOriginPrice( product, poco) else: _productBeforePrice = '暂不获取原价' #折扣 if CategaryNum < 2: _productDiscount = GetDiscount(product, poco) else: _productDiscount = '暂不获取折扣' print( message % (str(_productName), str(_productCurrentPrice), str(_productBeforePrice), str(_productSalesVolume), str(_productDiscount))) #入库 if _productCurrentPrice != -1: DbContext.SynchroProductInfo( storeId, _productName, int(_productSalesVolume), float(_productCurrentPrice), str(_productBeforePrice), str(_productDiscount)) except Exception as e: if ('UIObjectProxy' in repr(e)): continue #不存在的元素就直接下一个 else: print('获取商品信息异常:' + repr(e)) ErrorMessage = 'error' continue #判断是否滑动到底部 #爬完一页 print("爬完一批") Isbottom = poco( "com.sankuai.meituan.takeoutnew:id/noMoreView").exists() if Isbottom: BottomNum += 1 if BottomNum == 1: print('~~~~~~~~~~~~~~~~~~~~到达底部~~~~~~~~~~~~~~~~~~~~') else: print('~~~~~~~~~~~~~~~~~全部药品抓取完成~~~~~~~~~~~~~~~') break else: productNum = len(AllProductInfoList) #统计每次这个数量出现的频率,因为有的时候滑动到底部不会有到底的提示,所以当商品数量多次滑动不再增加的时候 #就认为商品抓取完毕 if CurrentProductNum != productNum: CurrentProductNum = productNum maxNum = 0 else: maxNum += 1 #滑动20次依旧没有商品新增,则认为到底了 if maxNum > 20: print('~~~~~~~~~~~~~~~~~程序判定滑动到了底部~~~~~~~~~~~~~~~') break else: device_screen = poco.get_screen_size() device_x = device_screen[0] device_y = device_screen[1] if device_y > 1600 and device_x > 1080: DeviceType = 2 else: DeviceType = 1 if DeviceType == 1: poco.swipe([0.5, 0.8], [0.5, 0.2], duration=0.2) else: poco.swipe([0.5, 0.8], [0.5, 0.3], duration=0.4) pass except PocoNoSuchNodeException: continue except Exception as e: print('爬取商品信息异常:' + repr(e)) continue pass EndTime = datetime.datetime.now() print('本次抓取耗时:' + str(((EndTime - StartTime).seconds) / 60) + ' 爬取药品数量:' + str(len(AllProductInfoList))) #连续返回两次上一页 backPage = poco("com.sankuai.meituan.takeoutnew:id/img_back_light") if backPage.exists(): print('返回上一页') backPage.wait(waitTime).click() backPage.wait(waitTime).click() else: device.keyevent("4") device.keyevent("4") return ErrorMessage
break else: break pass else: continue if isRunning: break except psutil.NoSuchProcess: pass if isRunning: pass else: # 判断程序是否是自己停掉,否则就要重启 DbContext = DbHelper() mode = DbContext.GetDeviceRunningMode(DeviceNum) if mode == 1: taskList = DbContext.GetDeviceTask(DeviceNum) elif mode == 2: taskList = DbContext.GetDeviceTaskByMode2() elif mode == 3: taskList = DbContext.GetDeviceTaskByMode3() elif mode == 5: taskList = [] elif mode == 6: taskList = [] if len(taskList) > 0: print( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Reatarting~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
def readCsv(targetCity): myCon = DbHelper() result = myCon.getlocation(targetCity) return result