def updateGeohash_taskbase():
sql = '''
select Lng,Lat,TaskId
from taskbase
where IsExcute = 4
'''
_base32 = '0123456789bcdefghjkmnpqrstuvwxyz'
_encode_map = {}
for i in range(len(_base32)):
_encode_map[i]=_base32[i]
DbContext = DbHelper()
result = DbContext.Query(sql)
for item in result:
Lng = item['Lng']
Lat = item['Lat']
TaskId = item['TaskId']
geohash = getGeoHashByLngLat(float(Lat),float(Lng),_encode_map)
sql = '''
update taskbase
set GenHash = '%s'
where TaskId = %d
'''
sql = sql % (geohash,TaskId)
DbContext.Update(sql)
print(str(TaskId) + '更新完')
pass
def updateGeohash():
sql = '''
select Lng,Lat,shopName,address
from shop
where address = '上海市浦东新区万祥镇严木桥路96号1-2层'
'''
_base32 = '0123456789bcdefghjkmnpqrstuvwxyz'
_encode_map = {}
for i in range(len(_base32)):
_encode_map[i]=_base32[i]
DbContext = DbHelper()
result = DbContext.Query(sql)
for item in result:
Lng = item['Lng']
Lat = item['Lat']
shopname = item['shopName']
address = item['address']
geohash = getGeoHashByLngLat(float(Lat),float(Lng),_encode_map)
sql = '''
update shop
set AddressGeohash = '%s'
where shopName = '%s' and address = '%s'
'''
sql = sql % (geohash,shopname,address)
DbContext.Update(sql)
pass
def SearchCatchStore(storeName: str, poco, device, DeviceNum: str, DeviceType: int):
# 查找到该门店的相关信息
AllClassifyInputClickNum = 0 # 大类进入点击小类的次数
IsAllClassifyInput = False # 是否从大类搜索进入(默认不是)
DbContext = DbHelper()
# result = DbContext.GetStorePoint(storeId)
result = DbContext.GetStorePointForName(storeName)
if len(result) == 1:
# address字段可能找不到这个店，用AnchorPoint才能找到这个店
strs = str(result[0]['AnchorPoint']).split(';')
StoreAddress = strs[0]
StoreName = result[0]['shopName']
StoreCity = result[0]['City']
storeId = result[0]['mtWmPoiId']
IsAddress = SwithPosition(poco, StoreAddress, StoreCity, storeName, 0) # 切换定位
isExists = getStore(IsAddress, poco, DeviceNum,
IsAllClassifyInput, StoreName, device)
if(not isExists):
BackHomePage(poco, DbContext, DeviceNum, device) # 返回首页
StoreAddress = result[0]['address']
IsAddress = SwithPosition(poco, StoreAddress, StoreCity, storeName, 0) # 切换定位
isExists = getStore(IsAddress, poco, DeviceNum, IsAllClassifyInput, StoreName, device)
if(not isExists):
# DbContext.AddLog(DeviceNum, 2, '爬取['+StoreName+']失败，没找到店铺')
print('该店铺不存在！')
pass
def BatchSendMessage(self):
sql = '''
update task
set IsExcute = 1
where IsExcute = 2
'''
DbContext = DbHelper()
DbContext.Update(sql)
pass
def delStore(self):
DbContext = DbHelper()
storelist = open("log.txt", mode='r')
for index in range(16901):
line = next(storelist)
delline = str(line).split(',')[0]
relust = DbContext.deleteStore(delline)
print("删除第%d行 - %s" % (index, delline) + str(relust))
pass
def getAddressFromGaoDe():
DbContext = DbHelper()
sql = """
select Lng,Lat
from address
where city like '%重庆%' and Lng is not null and Lat is not null and RepresentativeAdress is null
limit 5500
"""
allLngLat = DbContext.Query(sql)
key = '929d86bb3c93b6895554459ab7893171'
print(str(len(allLngLat)))
for item in allLngLat:
try:
#print(str(float(item["Lng"])),str(float(item["Lat"])))
#url = url % (key,float(),float(item["Lat"]))
url = 'https://restapi.amap.com/v3/geocode/regeo?key=' + key + '&location=' + str(item["Lng"]) + ',' + str(item["Lat"]) + '&poitype=&radius=1000&extensions=base&batch=false&roadlevel=1'
req = rq.get(url)
dataDic = json.loads(req.text)
address = ''
businessaddress = ''
priority = 3
if int(dataDic['status']) == 1:
baseaddress = str(dataDic['regeocode']['formatted_address']) #基础地址
if 'addressComponent' in dataDic['regeocode']:
if 'businessAreas' in dataDic['regeocode']['addressComponent']:
if len(dataDic['regeocode']['addressComponent']['businessAreas']) > 0:
result = dataDic['regeocode']['addressComponent']['businessAreas'][0]
print(dataDic['regeocode']['addressComponent']['businessAreas'])
if len(result) > 0:
if 'name' in result:
print(222)
businessaddress = ' ' + result["name"]
priority = 1
address = baseaddress + businessaddress
else:
address = ''
continue
if address != '' and address != '[]':
address = address.replace("'","")
UpdateAddress(float(item["Lng"]),float(item["Lat"]),str(address),priority,DbContext)
print(address)
except Exception as e:
if 'not all arguments converted during string formatting' in repr(e):
continue
else:
print('出错：'+ repr(e))
break
pass
pass
def __GoHeavy(self, taskid, execid, sendMessageList):
try:
r = CacheData()
for msg in sendMessageList:
if not r.CurrentlyData(taskid, execid, msg['Id']):
#去掉重复的
sendMessageList.remove(msg)
#判断是否要清除缓存
r.ActiveClear(taskid, execid)
except Exception as e:
DbContext = DbHelper()
DbContext.AddLog(
'', 4,
'redis去重异常异常：' + repr(e).replace("'", "").replace("\"", ""))
del DbContext
return sendMessageList
def sendErrorMessage(self, message):
try:
#创建通道
channel = self._conn.channel()
# channel.queue_declare(queue=self.__QueueName)
channel.basic_publish(exchange='',
routing_key=self.__QueueName,
body=message,
properties=pika.BasicProperties(
delivery_mode=2, ))
except Exception as e:
DbContext = DbHelper()
DbContext.AddLog(
'', 4,
'发送队列出错数据异常：' + repr(e).replace("'", "").replace("\"", ""))
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
print('发送队列出错数据异常：' + repr(e).replace("'", "").replace("\"", ""))
del DbContext
def ActiveClear(self, taskid, execid):
try:
sql = '''
select count(*) as num
from task
where TaskTag = '%s' and Exec = '%s' and IsExcute < 3
''' % (str(taskid), str(execid))
DbContext = DbHelper()
result = DbContext.Query(sql)
#等于1说明当前只剩下这个任务没有回传,因此可以清除
if result[0]['num'] == 1:
#清除这个set
SetName = str(taskid) + str(execid)
members = self.__conn.smembers(SetName)
for mem in members:
self.__conn.srem(SetName, mem)
del DbContext
except Exception as e:
print(repr(e))
pass
def sendMessage(self, taskId, messageList):
try:
message = self.__DealSendMessage(taskId, messageList)
#创建通道
channel = self._conn.channel()
#channel.queue_declare(queue=self.__QueueName,durable=True)
EncodeTextMsg = EncodeText(message)
channel.basic_publish(exchange='',
routing_key=self.__QueueName,
body=EncodeTextMsg,
properties=pika.BasicProperties(
delivery_mode=2, ))
except Exception as e:
DbContext = DbHelper()
DbContext.AddLog(
'', 4, '发送数据异常：' + repr(e).replace("'", "").replace("\"", ""))
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
print('发送队列出错数据异常：' + repr(e).replace("'", "").replace("\"", ""))
del DbContext
return False
else:
return True
def __DealSendMessage(self, taskId, sendMessageList):
Result = str(sendMessageList)
Data = {}
Data['State'] = 1
DbContext = DbHelper()
sql = '''
select Id,StoreId,Lng,Lat,GeoHash,Province,CityCode,District,TaskTag,Exec
from task
where TaskId = %d
''' % (int(taskId))
receiveMsgList = DbContext.Query(sql)
receiveMsgDict = receiveMsgList[0]
Data['Id'] = receiveMsgDict['Id']
Data['StoreId'] = receiveMsgDict['StoreId']
Data['Lng'] = receiveMsgDict['Lng']
Data['Lat'] = receiveMsgDict['Lat']
Data['GeoHash'] = receiveMsgDict['GeoHash']
Data['Platform'] = 'mt'
Data['Province'] = receiveMsgDict['Province']
Data['City'] = receiveMsgDict['CityCode']
Data['District'] = receiveMsgDict['District']
Data['Task'] = receiveMsgDict['TaskTag']
Data['Exec'] = receiveMsgDict['Exec']
Data['Type'] = 'StoreGet'
if len(sendMessageList) > 0:
Data['Exists'] = 1
sendMessageList = self.__GoHeavy(Data['Task'], Data['Exec'],
sendMessageList)
if len(sendMessageList) > 0:
Data['Exists'] = 1
else:
Data['Exists'] = 0
Result = str(sendMessageList)
else:
Data['Exists'] = 0
Data['Result'] = Result
return str(Data)
def getLngLatByAddress():
key = '929d86bb3c93b6895554459ab7893171'
sql = '''
select shopName,city,address
from shop s
where s.Genhash is null
'''
try:
DbContext = DbHelper()
Alladdress = DbContext.Query(sql)
for address in Alladdress:
_shopname = address['shopName']
_city = address['city']
_address = address['address']
url = 'https://restapi.amap.com/v3/geocode/geo?key='+ key +'&address=' + _address +'&city=' + _city
req = rq.get(url)
dataDic = json.loads(req.text)
if int(dataDic['status']) == 1:
geocodes = dataDic['geocodes']
if len(geocodes) > 0:
location = geocodes[0]['location'].split(",")
if len(location) > 0:
Lng = location[0]
Lat = location[1]
sql = '''
update shop
set Lng = '%s',
Lat = '%s'
where shopName = '%s' and address = '%s'
'''
sql = sql % (str(Lng),str(Lat),_shopname,_address)
DbContext.Update(sql)
print('已更新【'+ _shopname +'】')
pass
except Exception as e:
print(repr(e))
pass
def getLngLatByAddress_task():
key = '929d86bb3c93b6895554459ab7893171'
sql = '''
select TaskId,Address
from taskbase t
where t.IsExcute = 4
'''
try:
DbContext = DbHelper()
Alladdress = DbContext.Query(sql)
for address in Alladdress:
_TaskId = address['TaskId']
_address = address['Address']
url = 'https://restapi.amap.com/v3/geocode/geo?key='+ key +'&address=' + _address +'&city=福州'
req = rq.get(url)
dataDic = json.loads(req.text)
if int(dataDic['status']) == 1:
geocodes = dataDic['geocodes']
if len(geocodes) > 0:
location = geocodes[0]['location'].split(",")
if len(location) > 0:
Lng = location[0]
Lat = location[1]
sql = '''
update taskbase
set Lng = '%s',
Lat = '%s'
where TaskId = %d
'''
sql = sql % (str(Lng),str(Lat),_TaskId)
DbContext.Update(sql)
print('已更新【'+ _address +'】')
pass
except Exception as e:
print(repr(e))
pass
def RunningCheck():
print(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 设备运行情况检测 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
RunDevice = []
# 检查设备程序是否在运行
for proc in psutil.process_iter():
try:
pinfo = proc.as_dict(attrs=['cmdline'])
if type(pinfo['cmdline']).__name__ != 'NoneType':
if len(pinfo['cmdline']) > 0:
# 格式['python','main.py']
if '/bin/sh' == pinfo['cmdline'][0] and pinfo['cmdline'][
1] == '-c':
command = str(pinfo['cmdline'][2])
print(command)
command = command.replace(".py", "").replace(
"PyDaemon",
"").replace('./',
"").replace(" ", "").replace("-d", "")
if 'python3.7' in command:
deviceNum = command.replace("python3.7", "")[:16]
if deviceNum not in RunDevice:
RunDevice.append(deviceNum)
pass
except psutil.NoSuchProcess as e:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 检测设备状态异常 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
print(repr(e))
continue
# 这里会限制只能重启8台
if len(RunDevice) < 9:
# 获取没有运行的设备
DbContext = DbHelper()
DeviceDict = [
'5LM0216902001108', '5LM0216910000994', '5LM0216B03001264',
'APU0216408028484', 'DLQ0216630004610', 'E4J4C17405011422',
'DLQ0216729004546'
]
DeviceList = []
for devicenum in DeviceDict:
DeviceList.append(devicenum)
NotRunningDevice = list(set(DeviceList).difference(set(RunDevice)))
if len(NotRunningDevice) > 0:
# 启动没有运行的设备的Daemon
for deviceNum in NotRunningDevice:
try:
os.popen('python3.7 ' + deviceNum +
'.py >> /root/airtest/log/Device/Runniglog' +
datetime.datetime.now().strftime("%Y%m%d") +
'.log')
DbContext.AddLog(deviceNum, 1,
'启动设备[' + deviceNum + ']执行任务')
except Exception as e:
print('\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 重启设备[' +
deviceNum +
']异常 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
DbContext.AddLog(
deviceNum, 3, '重启设备[' + deviceNum + ']异常 ' +
repr(e).replace("'", ""))
pass
def StartCapture(poco, AllPosition, DeviceType, TargetCity, DeviceNum,
cityCode, device):
CloseUpDateInfo(poco)
IsAllClassifyInput = False # 是否从大类搜索进入(默认不是)
AllClassifyInputClickNum = 0 # 大类进入点击小类的次数
currentTaskResult = [] # 存放本次任务抓取的门店信息
if poco(text="美团外卖").exists():
poco(text="美团外卖").click()
if DeviceNum == 'E4J4C17405011422':
poco.swipe([0.5, 0.5], [0.5, 0.6], duration=0.3)
IsAddress = SwithPosition(poco, AllPosition[0]['RepresentativeAdress'],
TargetCity, DeviceNum, 0) # 第一次切换定位
if not IsAddress:
return currentTaskResult, False
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
try:
if poco(text="送药上门").exists():
poco(text="送药上门").wait(waitTime).click([0, 0])
else:
if DeviceNum == 'E4J4C17405011422':
poco.swipe([0.5, 0.6], [0.5, 0.5], duration=0.3)
result = SearchDrugPage(poco, device, DeviceNum)
if not result:
print(DeviceNum)
if poco(text="美食").exists():
IsAddress = False
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 第一个定位点无送药上门 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
else:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 被反爬检测到了 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
return currentTaskResult, True
else:
IsAllClassifyInput = True
except PocoNoSuchNodeException:
print(DeviceNum)
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 第一个定位点无送药上门f o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
IsAddress = False
pass
StoreList = [] #只存储当前定位的门店名称
SwitchNum = 0 #定位切换次数
DbContext = DbHelper()
for addressDic in AllPosition: #结构改了以后AllPosition只会有一条数据
address = addressDic['RepresentativeAdress'] #标志性坐标点
addressGenhash = addressDic['Genhash'] #地理散列值
print('===========================【' + DeviceNum + '】抓取【' + address +
'】开始===========================')
# 第一次进来不需要切换定位
if SwitchNum > 0:
StoreList.clear() #切换一次地址就将列表清空
IsAddress = SwithPosition(poco, address, TargetCity,
SwitchNum) #切换定位
if not IsAddress:
print('===========================【' + DeviceNum + '】抓取【' +
address + '】完成 无店铺===========================\n')
continue
SongYaoNum = 0
while True:
if poco(text="送药上门").exists():
poco(text="送药上门").wait(waitTime).click([0, 0])
break
SongYaoNum += 1
time.sleep(1)
if SongYaoNum >= 3:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 该定位点无送药上门 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
IsAddress = False
break
SwitchNum += 1
if not IsAddress:
print('===========================【' + DeviceNum + '】抓取【' +
address + '】完成 无店铺===========================\n')
continue
storelenNum = 0
swipeNume = 0
# 第一次大幅度滑动
if DeviceType == 1:
poco.swipe([0.2, 0.9], [0.2, 0.45], duration=0.3)
else:
poco.swipe([0.4, 0.9], [0.4, 0.75], duration=0.3)
if IsAllClassifyInput:
if poco(text="常用药品").exists():
poco(text="常用药品").click()
AllClassifyInputClickNum += 1
# 按照销量排序
if poco(text='销量').exists():
poco(text='销量').wait(waitTime).click()
# 当前循环是一个定位点的门店
backPage = poco("com.sankuai.meituan.takeoutnew:id/iv_back")
bottomElement = poco("com.sankuai.meituan.takeoutnew:id/noMoreView")
while True:
# 判断程序是否被紧急置停
mode = DbContext.GetDeviceRunningMode(DeviceNum)
if mode == 5:
device.keyevent("4")
return currentTaskResult, True
AllStore = poco(
"com.sankuai.meituan.takeoutnew:id/fl_fragment_container"
).offspring(
"com.sankuai.meituan.takeoutnew:id/pull_to_refresh_view"
).offspring(
"com.sankuai.meituan.takeoutnew:id/viewpager_content"
).offspring(
"com.sankuai.meituan.takeoutnew:id/wm_st_poi_channel_list"
).child("android.widget.FrameLayout").wait(waitTime)
i = 0
if len(AllStore) > 0:
# print('当前列表展示的门店数量：'+ str(len(AllStore)))
circleIsFail = False
for store in AllStore:
if i == 0:
i += 1 #第一个是顶部筛选
continue
storeInfo = {}
try:
storeNameResult = GetStoreName(store)
if not storeNameResult[0]:
continue
storeName = storeNameResult[2]
# 销量、评分
Sell = '0'
Score = '评分未知'
# time_ScoreUI = storeNameResult[1].child("android.widget.LinearLayout")
# try:
# time = time_ScoreUI.offspring("com.sankuai.meituan.takeoutnew:id/txt_mt_delivery_time_info").exists()
# #如果配送时间不存在就是快递店就不爬
# if not time:
# print("-------------快递店不爬【"+storeName+"】---------------")
# continue
# except PocoNoSuchNodeException:
# print("-------------獲取配送時間異常---------------")
storeSellResult = GetStoreSell(storeNameResult[1])
if not storeSellResult[0]:
continue
else:
Sell = storeSellResult[2]
storeScoreResult = GetStoreScore(
storeSellResult[1])
if not storeScoreResult[0]:
continue
else:
Score = storeScoreResult[1]
# 该定位点存在且门店名称也存在,有且只有一个地址,那么就不需要继续爬门店地址,更新销量和评分数据即可
IsClickStore = False
if storeSellResult[0] and storeScoreResult[
0]: #销量和评分全部获取到在判断
# 同一定位地址,同店铺名字的不予考虑
if storeName in StoreList:
continue
else:
if '成人用品' in storeName or '情趣' in storeName: #成人用品店不抓取,干扰太大
continue
StoreList.append(storeName)
storeInfo['Name'] = storeName
if '暂无评分' in str(Score):
storeInfo['Score'] = 0
else:
storeInfo['Score'] = float(Score)
Sell_Str = str(Sell).replace("月售", "").replace(
"+", "").replace("件", "")
storeInfo['Sales'] = float(Sell_Str)
storeInfo['Phone'] = ''
storeInfo['Brand'] = ''
storeInfo['Created'] = datetime.datetime.now(
).strftime("%Y-%m-%d %H:%M:%S")
IsClickStore, addressinfo, mtWmPoiId = DbContext.GetStoreInfo(
storeName, str(Score),
str(Sell).replace("月售",
"").replace("+", "").replace(
"件", ""), addressGenhash,
TargetCity, address)
# 继续爬取门店地址
if IsClickStore:
# 根据店名+城市找地址
storeAddress, mtWmPoiId = DbContext.UpdateGeoHash(
storeName, cityCode, addressGenhash)
storeInfo['OriginAddress'] = storeAddress
storeInfo['Id'] = mtWmPoiId
if FaileAddress in storeAddress:
# 根据店名+城市获取地址失败在继续单击查找
store.click()
storeAddress = GetStoreInfo(
poco, DbContext, DeviceNum)
Sell = str(Sell).replace("月售", "").replace(
"+", "").replace("件", "")
storeAddress = storeAddress.replace(
"地址", "").replace("地址：", "")
if FaileAddress not in storeAddress:
mtWmPoiId = DbContext.InsertShop(
storeName, Score, storeAddress, Sell,
address, addressGenhash, TargetCity,
cityCode)
storeInfo['Id'] = mtWmPoiId
storeInfo['OriginAddress'] = storeAddress
else:
storeAddress = '不需要爬取地址'
storeInfo['OriginAddress'] = addressinfo
storeInfo['Id'] = mtWmPoiId
print(storeName + ' 【' + str(Sell) + '】 【评分】:' +
str(Score) + ' 【地址】:' + storeAddress + '\n')
if FaileAddress not in storeInfo[
'OriginAddress'] and len(storeInfo['Id']) != 0:
storeInfo['OriginAddress'] = str(
storeInfo['OriginAddress']).replace(
u'\xa0', u' ')
currentTaskResult.append(storeInfo)
''' 药品信息
CatchProductResult = ProductInfo.GetProduct(poco)
if CatchProductResult == 'error':
circleIsFail = True
break
'''
except PocoNoSuchNodeException:
continue
except Exception as e:
DbContext.AddLog(
DeviceNum, 3, '设备[' + DeviceNum + ']爬取门店列表异常：' +
repr(e).replace("'", "").replace("\"", ""))
continue
pass
if circleIsFail:
print('获取门店列表失败！')
break
# 判断是否滑动到底部
Isbottom = bottomElement.exists()
if Isbottom:
print('~~~~~~~~~~~~~~~本次定位的门店到底了~~~~~~~~~~~~~~~')
if not ClickClassify(poco, AllClassifyInputClickNum):
if backPage.exists():
backPage.wait(waitTime).click()
else:
device.keyevent("4")
print('【因为抓取门店到底而结束本次定位查询】')
if IsAllClassifyInput:
device.keyevent("4")
break
else:
AllClassifyInputClickNum += 1
continue
else:
# 当前页面展示的门店数量
if len(AllStore) > 4:
if DeviceType == 1:
poco.swipe([0.5, 0.8], [0.5, 0.35], duration=0.3)
else:
poco.swipe([0.5, 0.8], [0.5, 0.5], duration=0.3)
else:
# 手机展示不会超过4 + 1个 P20展示5+1个
if DeviceType == 1:
poco.swipe([0.5, 0.8], [0.5, 0.45], duration=0.3)
else:
poco.swipe([0.5, 0.8], [0.5, 0.65], duration=0.3)
swipeNume += 1
# 滑动了40次(至少是120家店) 且门店数量还是小于一页，那么认为这个也没滑动到底且没有门店滑动到底的提示
if swipeNume > 40:
if len(StoreList) < 6 or len(AllStore) < 4:
if not ClickClassify(poco,
AllClassifyInputClickNum):
if backPage.exists():
backPage.wait(waitTime).click()
else:
device.keyevent("4")
# 如果是大类进来的，还要在返回一次
if IsAllClassifyInput:
device.keyevent("4")
print('认为这个也没滑动到底且没有门店滑动到底的提示而退出')
break
else:
AllClassifyInputClickNum += 1
continue
else:
# 判断定位点附近是否有门店
if poco(text='该定位下暂无服务商家，请切换地址').exists():
print('~~~~~~~~~~~~~~~~该定位下暂无服务商家，请切换地址~~~~~~~~~~~~~~~~')
if not ClickClassify(poco, AllClassifyInputClickNum):
if backPage.exists():
backPage.wait(waitTime).click()
else:
device.keyevent("4")
if IsAllClassifyInput:
device.keyevent("4")
break
else:
AllClassifyInputClickNum += 1
continue
# 跳到登录界面需要退出
if poco(text='获取短信验证码').exists():
device.keyevent("4")
continue
storelenNum += 1
if storelenNum > 5:
print('【因为获取门店列表长度为0而结束本次定位查询】')
if not ClickClassify(poco, AllClassifyInputClickNum):
if backPage.exists():
backPage.wait(waitTime).click()
else:
device.keyevent("4")
if IsAllClassifyInput:
device.keyevent("4")
break
else:
AllClassifyInputClickNum += 1
continue
else:
if DeviceType == 1:
poco.swipe([0.5, 0.9], [0.5, 0.7], duration=0.3)
continue
pass
print('===========================【' + DeviceNum + '】抓取【' + address +
'】完成===========================\n')
return currentTaskResult, False
def main(DeviceNum):
# 多设备连接时,可指定设备编号
# python -m airtest run main.py --device Android://127.0.0.1:5037/0123456789ABCDEF
# device = Android('GWY0216C16002906')
# 设备类型 1 手机[720,1280][1080,1920] 2平板[1200,1920]
# DeviceNum = 'c5bac654'
NeedSwipe = ['CLB0218414001154', 'DLQ0216824000142', 'E4J4C17405011422',
'APU0216530000778', 'APU0216408028484', 'APU0216111008105']
try:
DbContext = DbHelper()
DeviceType = 0
device = Android(DeviceNum)
# device.adb.start_shell("su")
# device.adb.start_shell("wipe data")
# device.adb.start_shell("wipe cache")
# device.adb.start_cmd("adb reboot")
if '0123456789ABCDEF' not in DeviceNum:
device.wake() # 唤醒页面
poco = AndroidUiautomationPoco(device)
# if DeviceNum in NeedSwipe:
poco.swipe([0.4, 0.9], [0.4, 0.55], duration=0.1)
time.sleep(2)
# 复位一下,防止之前没有睡眠也滑动
poco.swipe([0.4, 0.45], [0.4, 0.9], duration=0.1)
else:
poco = AndroidUiautomationPoco(
use_airtest_input=True, screenshot_each_action=False)
# ClearMemory(device,poco,'')
# return
device_screen = poco.get_screen_size()
device_x = device_screen[0]
device_y = device_screen[1]
print(str(device_y), str(device_x))
if device_y > 1600 and device_x > 1080:
DeviceType = 2
else:
DeviceType = 1
if DeviceType == 0:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 设备型号无法确定 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
DbContext.AddLog(DeviceNum, 3, '设备[' + DeviceNum + ']型号无法确定')
return
elif DeviceType == 1:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 设备型号为手机 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n')
else:
print(
'\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 设备型号为平板 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n')
except Exception as e:
print('\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~o(╥﹏╥)o 设备连接异常 o(╥﹏╥)o~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
DbContext.AddLog(
DeviceNum, 3, "设备[" + DeviceNum + "]连接异常：" + repr(e).replace("'", ""))
else:
BackHomeStatus = BackHomePage(
poco, DbContext, DeviceNum, device) # 返回首页
if BackHomeStatus:
mode = DbContext.GetDeviceRunningMode(DeviceNum)
taskList = []
# 获取当前设备要抓取的城市
IsRunning = True
if mode == 1:
taskList = DbContext.GetDeviceTask(DeviceNum)
elif mode == 2:
taskList = DbContext.GetDeviceTaskByMode2()
elif mode == 3:
taskList = DbContext.GetDeviceTaskByMode3()
elif mode == 5:
IsRunning = False
elif mode == 6:
# 根据店名爬取
while True:
data = get_shopName()
id_name_addr_city = data[0]
shopid = id_name_addr_city[0]
storeName = id_name_addr_city[1]
addr = id_name_addr_city[2]
city = (id_name_addr_city[3])[:-1]
print(storeName, addr, city)
if data[1] == 3:
crawl_status_code(storeName, 3)
else:
crawl_status_code(storeName, 1)
search_store(storeName, poco, device, addr, city, shopid)
BackHomePage(poco, DbContext, DeviceNum, device)
crawl_status_code(storeName, 2)
if(len(taskList) == 1):
# 更新任务为运行中
DbContext.UpdateTaskStatus(
int(taskList[0]['TaskId']), 1, 0, mode)
while IsRunning:
BackHomeStatus = BackHomePage(
poco, DbContext, DeviceNum, device) # 返回首页
if BackHomeStatus:
for task in taskList:
# AddressList(task['TargetCity']) #获取未抓取的坐标点
AllPosition = [
{'RepresentativeAdress': task['RepresentativeAdress'], 'Genhash':task['Genhash']}]
taskId = task['TaskId']
cityCode = task['CityCode']
# 更新任务为运行中
DbContext.UpdateTaskStatus(int(taskId), 1, 0, mode)
# 计时
StartTime = datetime.datetime.now()
# 返回值需要写进队列
currentTaskResult, IsEmergencyStop = StartCapture(
poco, AllPosition, DeviceType, task['TargetCity'], DeviceNum, cityCode, device) # 抓取数据
# 没有紧急置停的情况下才完成后续的更新
if not IsEmergencyStop:
EndTime = datetime.datetime.now()
DbContext = DbHelper()
DbContext.AddLog(DeviceNum, 2, '设备[' + DeviceNum + ']本次抓取[' + AllPosition[0]['RepresentativeAdress'] + '] [' + str(
len(currentTaskResult)) + '] 家店,耗时：' + str(((EndTime - StartTime).seconds)/60))
# 更新任务为完成
DbContext.UpdateTaskStatus(
int(taskId), 2, len(currentTaskResult), mode)
# 将任务的执行结果回写到队列
if mode != 3:
Produce = SendMessage()
result = Produce.sendMessage(
taskId, currentTaskResult)
if result:
# 将任务状态改为已回写队列
DbContext.UpdateTaskStatus(
int(taskId), 3, len(currentTaskResult), mode)
else:
break
mode = DbContext.GetDeviceRunningMode(DeviceNum)
if len(taskList) > 0:
taskList.clear()
if mode == 1:
taskList = DbContext.GetDeviceTask(DeviceNum)
elif mode == 2:
taskList = DbContext.GetDeviceTaskByMode2()
elif mode == 3:
taskList = DbContext.GetDeviceTaskByMode3()
elif mode == 5:
print(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 紧急置停 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
break
if len(taskList) > 0:
print(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 抽取一条任务 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
else:
print(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~O(∩_∩)O 任务执行完毕 O(∩_∩)O~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
break
else:
DbContext.AddLog(
DeviceNum, 3, '设备[' + DeviceNum + ']返回首页异常')
break
else:
DbContext.AddLog(DeviceNum, 3, '设备[' + DeviceNum + ']返回首页异常')
def GetProduct(poco, device, storeId):
if (poco("com.sankuai.meituan.takeoutnew:id/category_recycler_view").
exists()):
poco("com.sankuai.meituan.takeoutnew:id/category_recycler_view").swipe(
[-0.8, 0], duration=0.5)
time.sleep(3) #睡眠一段时间等待页面加载
ContainAll = poco(text="全部分类")
IsContainAll = ContainAll.exists()
AllProductInfoList = []
ErrorMessage = 'success'
DbContext = DbHelper()
if IsContainAll:
#找到父节点
parent = ContainAll.parent()
parent.wait(waitTime).click() #全部分类界面
StartTime = datetime.datetime.now()
time.sleep(1) #睡眠一段时间等待页面加载
SlideNum = 0 #滑动次数
BottomNum = 0 #到达底部次数
CategaryNum = 0 #爬取的大类数目（一般药店第一个是折扣商品会包含折扣、原价信息,之后的几个大类下的商品则不会包含）
#用于存储抓取的商品数量
CurrentProductNum = 0
maxNum = 0
while True:
try:
# if SlideNum % 5 == 0:#滑动几次之后大概睡眠一会
# time.sleep(1) #睡眠一段时间等待页面加载
# else:
#poco.swipe([0.5,0.7],[0.5,0.4],duration = 0.3)
#判断是否出现释放展示下一类按钮
# nextCategary = poco("com.sankuai.meituan.takeoutnew:id/tv_refresh").wait(waitTime).exists()
# if nextCategary:
# print('===================当前药品大类抓取完成===================')
# CategaryNum += 1
# pass
# SlideNum += 1
#当前要抓取的大类所能看见的界面展示
'''
if nextCategary:
time.sleep(1)
nextCategary = False'''
time.sleep(1)
CurrentClassfiy = poco(
"com.sankuai.meituan.takeoutnew:id/layout_shop_root_scroll_container"
).offspring(
"com.sankuai.meituan.takeoutnew:id/priority_scrollview"
).offspring("com.sankuai.meituan.takeoutnew:id/poi_pinned_layout"
).child("android.support.v7.widget.RecyclerView"
).child("android.widget.FrameLayout")
if (len(CurrentClassfiy) == 0):
CurrentClassfiy = poco(
"com.sankuai.meituan.takeoutnew:id/recycler"
).offspring(
"com.sankuai.meituan.takeoutnew:id/ll_stickyfoodList_adapter_food_food"
)
if (len(CurrentClassfiy) == 0):
CurrentClassfiy = poco(
"com.sankuai.meituan.takeoutnew:id/recycler"
).offspring(
"com.sankuai.meituan.takeoutnew:id/ll_stickyfoodList_adapter_food_food"
)
if len(CurrentClassfiy) > 0:
message = '【名称】：%s 【售价】:%s 【原价】：%s 【销量】：%s 【折扣】：%s'
print("本批抓到" + str(len(CurrentClassfiy)) + "个")
for product in CurrentClassfiy:
try:
#商品信息
# if(product.offspring(name = 'com.sankuai.meituan.takeoutnew:id/ll_stickyfoodList_adapter_food_food').exists()):
# product =product.offspring(name = 'com.sankuai.meituan.takeoutnew:id/ll_stickyfoodList_adapter_food_food')
# if len(product)== 0:
# print("当前为非商品标签")
# continue
#名称
_productName = GetProductName(product, poco)
if _productName == '获取名称失败':
continue
if _productName in AllProductInfoList:
print("【" + _productName + "】商品已存在")
continue #之前爬取过的就不做处理
else:
# _productPrice = product.child(name = 'com.sankuai.meituan.takeoutnew:id/ll_stickysold_count_unit_price_original_price_fix').child(name = 'com.sankuai.meituan.takeoutnew:id/ll_price_layout')
#销量
_productSalesVolume = GetSale(product, poco)
#售价
_productCurrentPrice = GetPrice(product, poco)
if _productCurrentPrice != -1:
#只有拿到售价的才算爬到商品的信息
AllProductInfoList.append(_productName)
#原价
if CategaryNum < 2:
_productBeforePrice = GetOriginPrice(
product, poco)
else:
_productBeforePrice = '暂不获取原价'
#折扣
if CategaryNum < 2:
_productDiscount = GetDiscount(product, poco)
else:
_productDiscount = '暂不获取折扣'
print(
message %
(str(_productName), str(_productCurrentPrice),
str(_productBeforePrice),
str(_productSalesVolume),
str(_productDiscount)))
#入库
if _productCurrentPrice != -1:
DbContext.SynchroProductInfo(
storeId, _productName,
int(_productSalesVolume),
float(_productCurrentPrice),
str(_productBeforePrice),
str(_productDiscount))
except Exception as e:
if ('UIObjectProxy' in repr(e)):
continue #不存在的元素就直接下一个
else:
print('获取商品信息异常：' + repr(e))
ErrorMessage = 'error'
continue
#判断是否滑动到底部
#爬完一页
print("爬完一批")
Isbottom = poco(
"com.sankuai.meituan.takeoutnew:id/noMoreView").exists()
if Isbottom:
BottomNum += 1
if BottomNum == 1:
print('~~~~~~~~~~~~~~~~~~~~到达底部~~~~~~~~~~~~~~~~~~~~')
else:
print('~~~~~~~~~~~~~~~~~全部药品抓取完成~~~~~~~~~~~~~~~')
break
else:
productNum = len(AllProductInfoList)
#统计每次这个数量出现的频率,因为有的时候滑动到底部不会有到底的提示,所以当商品数量多次滑动不再增加的时候
#就认为商品抓取完毕
if CurrentProductNum != productNum:
CurrentProductNum = productNum
maxNum = 0
else:
maxNum += 1
#滑动20次依旧没有商品新增,则认为到底了
if maxNum > 20:
print('~~~~~~~~~~~~~~~~~程序判定滑动到了底部~~~~~~~~~~~~~~~')
break
else:
device_screen = poco.get_screen_size()
device_x = device_screen[0]
device_y = device_screen[1]
if device_y > 1600 and device_x > 1080:
DeviceType = 2
else:
DeviceType = 1
if DeviceType == 1:
poco.swipe([0.5, 0.8], [0.5, 0.2], duration=0.2)
else:
poco.swipe([0.5, 0.8], [0.5, 0.3], duration=0.4)
pass
except PocoNoSuchNodeException:
continue
except Exception as e:
print('爬取商品信息异常：' + repr(e))
continue
pass
EndTime = datetime.datetime.now()
print('本次抓取耗时：' + str(((EndTime - StartTime).seconds) / 60) + ' 爬取药品数量：' +
str(len(AllProductInfoList)))
#连续返回两次上一页
backPage = poco("com.sankuai.meituan.takeoutnew:id/img_back_light")
if backPage.exists():
print('返回上一页')
backPage.wait(waitTime).click()
backPage.wait(waitTime).click()
else:
device.keyevent("4")
device.keyevent("4")
return ErrorMessage
break
else:
break
pass
else:
continue
if isRunning:
break
except psutil.NoSuchProcess:
pass
if isRunning:
pass
else:
# 判断程序是否是自己停掉，否则就要重启
DbContext = DbHelper()
mode = DbContext.GetDeviceRunningMode(DeviceNum)
if mode == 1:
taskList = DbContext.GetDeviceTask(DeviceNum)
elif mode == 2:
taskList = DbContext.GetDeviceTaskByMode2()
elif mode == 3:
taskList = DbContext.GetDeviceTaskByMode3()
elif mode == 5:
taskList = []
elif mode == 6:
taskList = []
if len(taskList) > 0:
print(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Reatarting~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
def readCsv(targetCity):
myCon = DbHelper()
result = myCon.getlocation(targetCity)
return result
