def __GoHeavy(self, taskid, execid, sendMessageList):
try:
r = CacheData()
for msg in sendMessageList:
if not r.CurrentlyData(taskid, execid, msg['Id']):
#去掉重复的
sendMessageList.remove(msg)
#判断是否要清除缓存
r.ActiveClear(taskid, execid)
except Exception as e:
DbContext = DbHelper()
DbContext.AddLog(
'', 4,
'redis去重异常异常：' + repr(e).replace("'", "").replace("\"", ""))
del DbContext
return sendMessageList
def sendErrorMessage(self, message):
try:
#创建通道
channel = self._conn.channel()
# channel.queue_declare(queue=self.__QueueName)
channel.basic_publish(exchange='',
routing_key=self.__QueueName,
body=message,
properties=pika.BasicProperties(
delivery_mode=2, ))
except Exception as e:
DbContext = DbHelper()
DbContext.AddLog(
'', 4,
'发送队列出错数据异常：' + repr(e).replace("'", "").replace("\"", ""))
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
print('发送队列出错数据异常：' + repr(e).replace("'", "").replace("\"", ""))
del DbContext
def sendMessage(self, taskId, messageList):
try:
message = self.__DealSendMessage(taskId, messageList)
#创建通道
channel = self._conn.channel()
#channel.queue_declare(queue=self.__QueueName,durable=True)
EncodeTextMsg = EncodeText(message)
channel.basic_publish(exchange='',
routing_key=self.__QueueName,
body=EncodeTextMsg,
properties=pika.BasicProperties(
delivery_mode=2, ))
except Exception as e:
DbContext = DbHelper()
DbContext.AddLog(
'', 4, '发送数据异常：' + repr(e).replace("'", "").replace("\"", ""))
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
print('发送队列出错数据异常：' + repr(e).replace("'", "").replace("\"", ""))
del DbContext
return False
else:
return True
def RunningCheck():
print(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 设备运行情况检测 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
RunDevice = []
# 检查设备程序是否在运行
for proc in psutil.process_iter():
try:
pinfo = proc.as_dict(attrs=['cmdline'])
if type(pinfo['cmdline']).__name__ != 'NoneType':
if len(pinfo['cmdline']) > 0:
# 格式['python','main.py']
if '/bin/sh' == pinfo['cmdline'][0] and pinfo['cmdline'][
1] == '-c':
command = str(pinfo['cmdline'][2])
print(command)
command = command.replace(".py", "").replace(
"PyDaemon",
"").replace('./',
"").replace(" ", "").replace("-d", "")
if 'python3.7' in command:
deviceNum = command.replace("python3.7", "")[:16]
if deviceNum not in RunDevice:
RunDevice.append(deviceNum)
pass
except psutil.NoSuchProcess as e:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 检测设备状态异常 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
print(repr(e))
continue
# 这里会限制只能重启8台
if len(RunDevice) < 9:
# 获取没有运行的设备
DbContext = DbHelper()
DeviceDict = [
'5LM0216902001108', '5LM0216910000994', '5LM0216B03001264',
'APU0216408028484', 'DLQ0216630004610', 'E4J4C17405011422',
'DLQ0216729004546'
]
DeviceList = []
for devicenum in DeviceDict:
DeviceList.append(devicenum)
NotRunningDevice = list(set(DeviceList).difference(set(RunDevice)))
if len(NotRunningDevice) > 0:
# 启动没有运行的设备的Daemon
for deviceNum in NotRunningDevice:
try:
os.popen('python3.7 ' + deviceNum +
'.py >> /root/airtest/log/Device/Runniglog' +
datetime.datetime.now().strftime("%Y%m%d") +
'.log')
DbContext.AddLog(deviceNum, 1,
'启动设备[' + deviceNum + ']执行任务')
except Exception as e:
print('\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 重启设备[' +
deviceNum +
']异常 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
DbContext.AddLog(
deviceNum, 3, '重启设备[' + deviceNum + ']异常 ' +
repr(e).replace("'", ""))
pass
def StartCapture(poco, AllPosition, DeviceType, TargetCity, DeviceNum,
cityCode, device):
CloseUpDateInfo(poco)
IsAllClassifyInput = False # 是否从大类搜索进入(默认不是)
AllClassifyInputClickNum = 0 # 大类进入点击小类的次数
currentTaskResult = [] # 存放本次任务抓取的门店信息
if poco(text="美团外卖").exists():
poco(text="美团外卖").click()
if DeviceNum == 'E4J4C17405011422':
poco.swipe([0.5, 0.5], [0.5, 0.6], duration=0.3)
IsAddress = SwithPosition(poco, AllPosition[0]['RepresentativeAdress'],
TargetCity, DeviceNum, 0) # 第一次切换定位
if not IsAddress:
return currentTaskResult, False
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
try:
if poco(text="送药上门").exists():
poco(text="送药上门").wait(waitTime).click([0, 0])
else:
if DeviceNum == 'E4J4C17405011422':
poco.swipe([0.5, 0.6], [0.5, 0.5], duration=0.3)
result = SearchDrugPage(poco, device, DeviceNum)
if not result:
print(DeviceNum)
if poco(text="美食").exists():
IsAddress = False
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 第一个定位点无送药上门 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
else:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 被反爬检测到了 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
return currentTaskResult, True
else:
IsAllClassifyInput = True
except PocoNoSuchNodeException:
print(DeviceNum)
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 第一个定位点无送药上门f o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
IsAddress = False
pass
StoreList = [] #只存储当前定位的门店名称
SwitchNum = 0 #定位切换次数
DbContext = DbHelper()
for addressDic in AllPosition: #结构改了以后AllPosition只会有一条数据
address = addressDic['RepresentativeAdress'] #标志性坐标点
addressGenhash = addressDic['Genhash'] #地理散列值
print('===========================【' + DeviceNum + '】抓取【' + address +
'】开始===========================')
# 第一次进来不需要切换定位
if SwitchNum > 0:
StoreList.clear() #切换一次地址就将列表清空
IsAddress = SwithPosition(poco, address, TargetCity,
SwitchNum) #切换定位
if not IsAddress:
print('===========================【' + DeviceNum + '】抓取【' +
address + '】完成 无店铺===========================\n')
continue
SongYaoNum = 0
while True:
if poco(text="送药上门").exists():
poco(text="送药上门").wait(waitTime).click([0, 0])
break
SongYaoNum += 1
time.sleep(1)
if SongYaoNum >= 3:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 该定位点无送药上门 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
IsAddress = False
break
SwitchNum += 1
if not IsAddress:
print('===========================【' + DeviceNum + '】抓取【' +
address + '】完成 无店铺===========================\n')
continue
storelenNum = 0
swipeNume = 0
# 第一次大幅度滑动
if DeviceType == 1:
poco.swipe([0.2, 0.9], [0.2, 0.45], duration=0.3)
else:
poco.swipe([0.4, 0.9], [0.4, 0.75], duration=0.3)
if IsAllClassifyInput:
if poco(text="常用药品").exists():
poco(text="常用药品").click()
AllClassifyInputClickNum += 1
# 按照销量排序
if poco(text='销量').exists():
poco(text='销量').wait(waitTime).click()
# 当前循环是一个定位点的门店
backPage = poco("com.sankuai.meituan.takeoutnew:id/iv_back")
bottomElement = poco("com.sankuai.meituan.takeoutnew:id/noMoreView")
while True:
# 判断程序是否被紧急置停
mode = DbContext.GetDeviceRunningMode(DeviceNum)
if mode == 5:
device.keyevent("4")
return currentTaskResult, True
AllStore = poco(
"com.sankuai.meituan.takeoutnew:id/fl_fragment_container"
).offspring(
"com.sankuai.meituan.takeoutnew:id/pull_to_refresh_view"
).offspring(
"com.sankuai.meituan.takeoutnew:id/viewpager_content"
).offspring(
"com.sankuai.meituan.takeoutnew:id/wm_st_poi_channel_list"
).child("android.widget.FrameLayout").wait(waitTime)
i = 0
if len(AllStore) > 0:
# print('当前列表展示的门店数量：'+ str(len(AllStore)))
circleIsFail = False
for store in AllStore:
if i == 0:
i += 1 #第一个是顶部筛选
continue
storeInfo = {}
try:
storeNameResult = GetStoreName(store)
if not storeNameResult[0]:
continue
storeName = storeNameResult[2]
# 销量、评分
Sell = '0'
Score = '评分未知'
# time_ScoreUI = storeNameResult[1].child("android.widget.LinearLayout")
# try:
# time = time_ScoreUI.offspring("com.sankuai.meituan.takeoutnew:id/txt_mt_delivery_time_info").exists()
# #如果配送时间不存在就是快递店就不爬
# if not time:
# print("-------------快递店不爬【"+storeName+"】---------------")
# continue
# except PocoNoSuchNodeException:
# print("-------------獲取配送時間異常---------------")
storeSellResult = GetStoreSell(storeNameResult[1])
if not storeSellResult[0]:
continue
else:
Sell = storeSellResult[2]
storeScoreResult = GetStoreScore(
storeSellResult[1])
if not storeScoreResult[0]:
continue
else:
Score = storeScoreResult[1]
# 该定位点存在且门店名称也存在,有且只有一个地址,那么就不需要继续爬门店地址,更新销量和评分数据即可
IsClickStore = False
if storeSellResult[0] and storeScoreResult[
0]: #销量和评分全部获取到在判断
# 同一定位地址,同店铺名字的不予考虑
if storeName in StoreList:
continue
else:
if '成人用品' in storeName or '情趣' in storeName: #成人用品店不抓取,干扰太大
continue
StoreList.append(storeName)
storeInfo['Name'] = storeName
if '暂无评分' in str(Score):
storeInfo['Score'] = 0
else:
storeInfo['Score'] = float(Score)
Sell_Str = str(Sell).replace("月售", "").replace(
"+", "").replace("件", "")
storeInfo['Sales'] = float(Sell_Str)
storeInfo['Phone'] = ''
storeInfo['Brand'] = ''
storeInfo['Created'] = datetime.datetime.now(
).strftime("%Y-%m-%d %H:%M:%S")
IsClickStore, addressinfo, mtWmPoiId = DbContext.GetStoreInfo(
storeName, str(Score),
str(Sell).replace("月售",
"").replace("+", "").replace(
"件", ""), addressGenhash,
TargetCity, address)
# 继续爬取门店地址
if IsClickStore:
# 根据店名+城市找地址
storeAddress, mtWmPoiId = DbContext.UpdateGeoHash(
storeName, cityCode, addressGenhash)
storeInfo['OriginAddress'] = storeAddress
storeInfo['Id'] = mtWmPoiId
if FaileAddress in storeAddress:
# 根据店名+城市获取地址失败在继续单击查找
store.click()
storeAddress = GetStoreInfo(
poco, DbContext, DeviceNum)
Sell = str(Sell).replace("月售", "").replace(
"+", "").replace("件", "")
storeAddress = storeAddress.replace(
"地址", "").replace("地址：", "")
if FaileAddress not in storeAddress:
mtWmPoiId = DbContext.InsertShop(
storeName, Score, storeAddress, Sell,
address, addressGenhash, TargetCity,
cityCode)
storeInfo['Id'] = mtWmPoiId
storeInfo['OriginAddress'] = storeAddress
else:
storeAddress = '不需要爬取地址'
storeInfo['OriginAddress'] = addressinfo
storeInfo['Id'] = mtWmPoiId
print(storeName + ' 【' + str(Sell) + '】 【评分】:' +
str(Score) + ' 【地址】:' + storeAddress + '\n')
if FaileAddress not in storeInfo[
'OriginAddress'] and len(storeInfo['Id']) != 0:
storeInfo['OriginAddress'] = str(
storeInfo['OriginAddress']).replace(
u'\xa0', u' ')
currentTaskResult.append(storeInfo)
''' 药品信息
CatchProductResult = ProductInfo.GetProduct(poco)
if CatchProductResult == 'error':
circleIsFail = True
break
'''
except PocoNoSuchNodeException:
continue
except Exception as e:
DbContext.AddLog(
DeviceNum, 3, '设备[' + DeviceNum + ']爬取门店列表异常：' +
repr(e).replace("'", "").replace("\"", ""))
continue
pass
if circleIsFail:
print('获取门店列表失败！')
break
# 判断是否滑动到底部
Isbottom = bottomElement.exists()
if Isbottom:
print('~~~~~~~~~~~~~~~本次定位的门店到底了~~~~~~~~~~~~~~~')
if not ClickClassify(poco, AllClassifyInputClickNum):
if backPage.exists():
backPage.wait(waitTime).click()
else:
device.keyevent("4")
print('【因为抓取门店到底而结束本次定位查询】')
if IsAllClassifyInput:
device.keyevent("4")
break
else:
AllClassifyInputClickNum += 1
continue
else:
# 当前页面展示的门店数量
if len(AllStore) > 4:
if DeviceType == 1:
poco.swipe([0.5, 0.8], [0.5, 0.35], duration=0.3)
else:
poco.swipe([0.5, 0.8], [0.5, 0.5], duration=0.3)
else:
# 手机展示不会超过4 + 1个 P20展示5+1个
if DeviceType == 1:
poco.swipe([0.5, 0.8], [0.5, 0.45], duration=0.3)
else:
poco.swipe([0.5, 0.8], [0.5, 0.65], duration=0.3)
swipeNume += 1
# 滑动了40次(至少是120家店) 且门店数量还是小于一页，那么认为这个也没滑动到底且没有门店滑动到底的提示
if swipeNume > 40:
if len(StoreList) < 6 or len(AllStore) < 4:
if not ClickClassify(poco,
AllClassifyInputClickNum):
if backPage.exists():
backPage.wait(waitTime).click()
else:
device.keyevent("4")
# 如果是大类进来的，还要在返回一次
if IsAllClassifyInput:
device.keyevent("4")
print('认为这个也没滑动到底且没有门店滑动到底的提示而退出')
break
else:
AllClassifyInputClickNum += 1
continue
else:
# 判断定位点附近是否有门店
if poco(text='该定位下暂无服务商家，请切换地址').exists():
print('~~~~~~~~~~~~~~~~该定位下暂无服务商家，请切换地址~~~~~~~~~~~~~~~~')
if not ClickClassify(poco, AllClassifyInputClickNum):
if backPage.exists():
backPage.wait(waitTime).click()
else:
device.keyevent("4")
if IsAllClassifyInput:
device.keyevent("4")
break
else:
AllClassifyInputClickNum += 1
continue
# 跳到登录界面需要退出
if poco(text='获取短信验证码').exists():
device.keyevent("4")
continue
storelenNum += 1
if storelenNum > 5:
print('【因为获取门店列表长度为0而结束本次定位查询】')
if not ClickClassify(poco, AllClassifyInputClickNum):
if backPage.exists():
backPage.wait(waitTime).click()
else:
device.keyevent("4")
if IsAllClassifyInput:
device.keyevent("4")
break
else:
AllClassifyInputClickNum += 1
continue
else:
if DeviceType == 1:
poco.swipe([0.5, 0.9], [0.5, 0.7], duration=0.3)
continue
pass
print('===========================【' + DeviceNum + '】抓取【' + address +
'】完成===========================\n')
return currentTaskResult, False
def main(DeviceNum):
# 多设备连接时,可指定设备编号
# python -m airtest run main.py --device Android://127.0.0.1:5037/0123456789ABCDEF
# device = Android('GWY0216C16002906')
# 设备类型 1 手机[720,1280][1080,1920] 2平板[1200,1920]
# DeviceNum = 'c5bac654'
NeedSwipe = ['CLB0218414001154', 'DLQ0216824000142', 'E4J4C17405011422',
'APU0216530000778', 'APU0216408028484', 'APU0216111008105']
try:
DbContext = DbHelper()
DeviceType = 0
device = Android(DeviceNum)
# device.adb.start_shell("su")
# device.adb.start_shell("wipe data")
# device.adb.start_shell("wipe cache")
# device.adb.start_cmd("adb reboot")
if '0123456789ABCDEF' not in DeviceNum:
device.wake() # 唤醒页面
poco = AndroidUiautomationPoco(device)
# if DeviceNum in NeedSwipe:
poco.swipe([0.4, 0.9], [0.4, 0.55], duration=0.1)
time.sleep(2)
# 复位一下,防止之前没有睡眠也滑动
poco.swipe([0.4, 0.45], [0.4, 0.9], duration=0.1)
else:
poco = AndroidUiautomationPoco(
use_airtest_input=True, screenshot_each_action=False)
# ClearMemory(device,poco,'')
# return
device_screen = poco.get_screen_size()
device_x = device_screen[0]
device_y = device_screen[1]
print(str(device_y), str(device_x))
if device_y > 1600 and device_x > 1080:
DeviceType = 2
else:
DeviceType = 1
if DeviceType == 0:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 设备型号无法确定 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
DbContext.AddLog(DeviceNum, 3, '设备[' + DeviceNum + ']型号无法确定')
return
elif DeviceType == 1:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 设备型号为手机 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n')
else:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 设备型号为平板 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n')
except Exception as e:
print('\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 设备连接异常 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
DbContext.AddLog(
DeviceNum, 3, "设备[" + DeviceNum + "]连接异常：" + repr(e).replace("'", ""))
else:
BackHomeStatus = BackHomePage(
poco, DbContext, DeviceNum, device) # 返回首页
if BackHomeStatus:
mode = DbContext.GetDeviceRunningMode(DeviceNum)
taskList = []
# 获取当前设备要抓取的城市
IsRunning = True
if mode == 1:
taskList = DbContext.GetDeviceTask(DeviceNum)
elif mode == 2:
taskList = DbContext.GetDeviceTaskByMode2()
elif mode == 3:
taskList = DbContext.GetDeviceTaskByMode3()
elif mode == 5:
IsRunning = False
elif mode == 6:
# 根据店名爬取
while True:
data = get_shopName()
id_name_addr_city = data[0]
shopid = id_name_addr_city[0]
storeName = id_name_addr_city[1]
addr = id_name_addr_city[2]
city = (id_name_addr_city[3])[:-1]
print(storeName, addr, city)
if data[1] == 3:
crawl_status_code(storeName, 3)
else:
crawl_status_code(storeName, 1)
search_store(storeName, poco, device, addr, city, shopid)
BackHomePage(poco, DbContext, DeviceNum, device)
crawl_status_code(storeName, 2)
if(len(taskList) == 1):
# 更新任务为运行中
DbContext.UpdateTaskStatus(
int(taskList[0]['TaskId']), 1, 0, mode)
while IsRunning:
BackHomeStatus = BackHomePage(
poco, DbContext, DeviceNum, device) # 返回首页
if BackHomeStatus:
for task in taskList:
# AddressList(task['TargetCity']) #获取未抓取的坐标点
AllPosition = [
{'RepresentativeAdress': task['RepresentativeAdress'], 'Genhash':task['Genhash']}]
taskId = task['TaskId']
cityCode = task['CityCode']
# 更新任务为运行中
DbContext.UpdateTaskStatus(int(taskId), 1, 0, mode)
# 计时
StartTime = datetime.datetime.now()
# 返回值需要写进队列
currentTaskResult, IsEmergencyStop = StartCapture(
poco, AllPosition, DeviceType, task['TargetCity'], DeviceNum, cityCode, device) # 抓取数据
# 没有紧急置停的情况下才完成后续的更新
if not IsEmergencyStop:
EndTime = datetime.datetime.now()
DbContext = DbHelper()
DbContext.AddLog(DeviceNum, 2, '设备[' + DeviceNum + ']本次抓取[' + AllPosition[0]['RepresentativeAdress'] + '] [' + str(
len(currentTaskResult)) + '] 家店,耗时：' + str(((EndTime - StartTime).seconds)/60))
# 更新任务为完成
DbContext.UpdateTaskStatus(
int(taskId), 2, len(currentTaskResult), mode)
# 将任务的执行结果回写到队列
if mode != 3:
Produce = SendMessage()
result = Produce.sendMessage(
taskId, currentTaskResult)
if result:
# 将任务状态改为已回写队列
DbContext.UpdateTaskStatus(
int(taskId), 3, len(currentTaskResult), mode)
else:
break
mode = DbContext.GetDeviceRunningMode(DeviceNum)
if len(taskList) > 0:
taskList.clear()
if mode == 1:
taskList = DbContext.GetDeviceTask(DeviceNum)
elif mode == 2:
taskList = DbContext.GetDeviceTaskByMode2()
elif mode == 3:
taskList = DbContext.GetDeviceTaskByMode3()
elif mode == 5:
print(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 紧急置停 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
break
if len(taskList) > 0:
print(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 抽取一条任务 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
else:
print(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 任务执行完毕 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
break
else:
DbContext.AddLog(
DeviceNum, 3, '设备[' + DeviceNum + ']返回首页异常')
break
else:
DbContext.AddLog(DeviceNum, 3, '设备[' + DeviceNum + ']返回首页异常')
