setting.headers["Connection"] = "keep-alive"
downloader = Downloader(setting=setting)
post_data = {}
post_data["geoType"] = "2"
post_data["cityId"] = "1"
post_data["secondCategoryId"] = ""
post_data["start"] = "0"
post_data["queryType"] = "11002"
post_data["keyword"] = kwarg["keyword"]
post_data["categoryType"] = ""
post_data["entranceId"] = "0"
post_data[
"uuid"] = "16d14d36a98c8-0e53a36f17ba0e-1a201708-1fa400-16d14d36a98c8"
post_data["platform"] = "3"
post_data["partner"] = "4"
post_data[
"originUrl"] = "https://h5.waimai.meituan.com/waimai/mindex/searchresults?queryType=11002&entranceId=0&keyword=%E5%B9%B2%E6%8D%9E%E8%9E%BA%E8%9B%B3%E7%B2%89&qwTypeId=11002&mode=1"
post_data["riskLevel"] = "71"
post_data["optimusCode"] = "10"
post_data["wm_latitude"] = "0"
post_data["wm_longitude"] = "0"
post_data["wm_actual_latitude"] = kwarg["lat"]
post_data["wm_actual_longitude"] = kwarg["lng"]
post_data[
"openh5_uuid"] = "16d14d36a98c8-0e53a36f17ba0e-1a201708-1fa400-16d14d36a98c8"
post_data[
"_token"] = "eJxNUGmvokAQ/C8kvC8aYRCHweRlI5fiE08OdbMfkFtBrsEBN/vfF/e9zW7S6aqurlQ6/ZOqdJ+aAhYILBhSj6CiphQYsSNIDSlc95sJRCxAggjFCTekvP80xLKQF4fUpbIVavp9AvmhIMAfL2Hfz/+Ef4zj+3o59N5AxRgX9ZRh4smIuEnmJqMsSHDj3kdenjGfEpMldz9omTpwKy+ugrpJcf2tbIKqM7sieAeAZbm34I4r9+4Fuv/Ovt2CjuSV/06rE1oSaYmjVUgjhRZVWkWvLs3+EImWxrQqvAxIfCvJK68P+EzMcr8P759A9edmZn9uj7cvdL8Q/52N/mu9t06ie8+CZZdaFt6Q58x6zJrDw8LXlrkl+1NirZbSiVR51zm4Wl80s4ULVpYJP7P1rWWfyQIqC6Kmq72fdXVIkIC4y/yM7LEhGTA84iZXLQ191BVqrFMhzT2QzVa7JIj2RaR0YvSwP6CpMAEumG0mjrMwDLtBZYukzKXNDmXHQ1h7Wnx4cLvYHOiJH0XqdbUmh3ItmLUcj2WD1StNKURyb5W0wNf9HGisYCfOLZ3LaWs99Q3gUo/buCJuwQRpeWsd160bzU9m3ux5pxyHnOkQfCbl9jLDoXZUtxfbdUVncSaG6hW6D4t6j4SPmbhLS8XYjTOdTxprIifbwXMjLGT+3MXXj1zgY/MhdSDcAmPHGaI7c9aAXGSzAp69bg3gQDO+DxrArfy7qihATlXPwppaOjwyFo4iePMbTpoQXnhYBshYzS2Wq7VlY2pB1MLI3sMtM0+XT7yz8vNpDE+BAuBC1hTfIclZRLxfXRvq129XkPn8"
print(
downloader.post("http://i.waimai.meituan.com/openh5/search/poi",
data=post_data).text)
time.sleep(3)
post_data[
"uuid"] = "2E42332BF3FA3E12F5CFCFA99E799888E8883DB85EF252B10CB184761FFDB340"
post_data["platform"] = "3"
post_data["partner"] = "4"
post_data[
"originUrl"] = "https://h5.waimai.meituan.com/waimai/mindex/menu?dpShopId=&mtShopId=880752952881352&utm_source=&source=shoplist&initialLat=&initialLng=&actualLat=23.125764&actualLng=113.334692"
post_data["riskLevel"] = "71"
post_data["optimusCode"] = "10"
post_data["wm_latitude"] = "24777226"
post_data["wm_longitude"] = "113678685"
post_data["wm_actual_latitude"] = "0"
post_data["wm_actual_longitude"] = "0"
post_data[
"openh5_uuid"] = "2E42332BF3FA3E12F5CFCFA99E799888E8883DB85EF252B10CB184761FFDB340"
post_data["_token"] = (
"eJxV0mmPqjoYAOD/wge/SKQtlGUSc6MCKo4LIOp4MrlhlUWWkYqjJ+e/n6ojuTch6cO7UQq/mdM0YN4ggBKALNOEJ+aNgT3Q"
"ExmWITXNYFEGiMcSAAIt8P8Xw7KksIx32qjM2y8sK6yC4Oc9YNH7X1BBgIVABp/syzz+ZJFAr3vVlBYxMSFV/cZxMe5d3CR3"
"k14eJuTsFj2/zLlniMuTIgi/uTwszv8ElR2X1TTod3LyI1kGEkYKRrIMeYw6Z5L/W5fnkx/2Oz9rTSuPSU06SZGQxD2+u6Tf"
"ujj0O65PH/oII74HEZZE4RWjaQj5Hs8LooLosTB08/n6vnkeCCzEAo3xgG+FWsFW9OUxpkKKQiU+hKnkh2iHCO+SZSrhoXtM"
"ukuidRJ6iE6RHr2iyEIZPESfK/OtpJcU2Aq3emURAK1QK6GV2Ep+CbYd8DmZfkX47MVUzx1IVM8pCtVjigSp5PuRZfcjo6v7"
"36Nj1enmniSv5Jz+f7SvTg4FVWhcSabrzeE6cOJIeZ/wsTOwdN+s9l+LbWHH151X19F4tplfB/ZS8BqlC6VNdU755rBTz4I/"
"VSKDdOXSz21tmq6H6co5mS6p/A8gpPZiXhvjiuwWoSDlH4OZVsJLkn0Lc1NZHq5Eu8SFMba0kQNcM8dmgCb6PhgFxtEyZvZm"
"sg6z+rQeV1uYb5VbtJxhKTPL0tADYXPYOOBjthva+jHJVvMFbGxS5vUwNfeNfhTxWsvmWISSh/ncMS/JuhGa9OBsC6siYwt4"
"Va2ayK2QcYP7mix5Z4+uzi4UJXGnX7SurYvDCC64zL4txtJ+K24sIwrMblhGjuqKXyPPdzYQ2udJVW5jAXVHelfmv2f5/gC3"
"t/VC0/QF53sjS685rxly29iqs2/5C3kwWO2qwQisrr6mhWoYNWCsrzhVGdtLLFiT0eB9fi52hzQ81lF3EYYgnS1tP+vexs0q"
"UtV0Sub1HPCG6SJu0O8zf/4COoo2Jw==")
print(
downloader.post("http://i.waimai.meituan.com/openh5/poi/comments",
data=post_data).text)
def crawlShopList():
post_data_of_shop_list = {
"startIndex":
"0",
"sortId":
"0",
"multiFilterIds":
"",
"sliderSelectCode":
"",
"sliderSelectMin":
"",
"sliderSelectMax":
"",
"geoType":
"2",
"rankTraceId":
"",
"uuid":
"EDAF663058DC10DD9DA74BF73550627E0BA82B4963E47B9FAEA941817D9F2F1C",
"platform":
"3",
"partner":
"4",
"originUrl":
"http://h5.waimai.meituan.com/waimai/mindex/home",
"riskLevel":
"71",
"optimusCode":
"10",
"wm_latitude":
"23129112",
"wm_longitude":
"113264385",
"wm_actual_latitude":
"",
"wm_actual_longitude":
"",
"openh5_uuid":
"EDAF663058DC10DD9DA74BF73550627E0BA82B4963E47B9FAEA941817D9F2F1C",
"_token":
("eJx9kG2PmkAUhf/LJNsvEgcFBExMI7CyuAgi43SlaRoEVORtweFFmv73DjHN7oemyU3uuc+9J3Myv0BlhGA+YSciO2FAE1VgDiZ"
"jdjwDDCA3uhFms6nMT1lR5KYMCD6ziShLPAOOFdbA/DsviIzET38MYEfnD/Chpjyt4cKgB+BCyPscwoswbv048+NxFsWk9vNxUG"
"TwgWAW52HUDS2iif7rSItznH89FVUQLUhVR1+OfpD8rKt0MdieuOXTdEXrn2bKH5CKx5MPEQGaNkNDWklmRFkY5mSYafc/cUYz8"
"LAjf3cb+pE08S0+51RF6ztJNNKc+6XTibKpFPBy9XVXkZZn44gR26VeYGAEnzXdsjOukje2j965xhqFsJDEXlfs/F632esltOvO"
"sStNJtoqxkQxrboVUdmn2DvlrnfGjZGqfUIi9ohqf71ODzsRWeW9MkoRbRLop5aXYhHjbO+tdi+512FuH2n6i08kf9PqppMZt76"
"8nirdNYMRQc9q70nu2lFRWzpmYJ52d0vNBadYKdtgyX4rePXWXgsrLXGSBq+yPiMHp+wdQdhgAzrI6HxO6i3prDus8d48q1Fbuc"
"o1JDDRot3L27ZVJ4p0mJmd6/ZvWQ/rI7lAKLTs3jo0SBZsDY1mr9zOHB3Y0ybcpn2A+aDZh/YWLxcL8PsPSaDfFA=="
),
}
total = 0
while True:
# download 解析
res = Downloader.post(SHOP_SEARCH_URL.format(time_util.get_unix()),
post_data_of_shop_list,
setting=setting)
jsonObject = res.json()
# 判断是否请求结束
if total > 2000:
logger.info("商品列表超过两千，结束请求")
break
# 判断是否请求异常
if "data" not in jsonObject or not isinstance(jsonObject["data"],
dict):
logger.error("第[{}]次 登录失效 or 接口异常 ".format(
post_data_of_shop_list["startIndex"]))
post_data_of_shop_list["startIndex"] = str(
int(post_data_of_shop_list["startIndex"]) + 1)
continue
# 解析抽取
try:
for shop in jsonObject["data"]["shopList"]:
shop["_id"] = shop["mtWmPoiId"]
ttfDecrypt(shop)
meituanwaimai_shop_list.save(shop)
shop_list.add(shop["mtWmPoiId"])
total += len(jsonObject["data"]["shopList"])
# 写入数据库
logger.info("第[{}]次 请求，抓取数据 {}量".format(
post_data_of_shop_list["startIndex"],
len(jsonObject["data"]["shopList"])))
except Exception as e:
#
logger.error("第[{}]次 解析失败 或 插入失败".format(
post_data_of_shop_list["startIndex"]))
traceback.print_exc()
post_data_of_shop_list["startIndex"] = str(
int(post_data_of_shop_list["startIndex"]) + 1)
time_util.sleep(5)
post_data["source"] = "shoplist"
post_data["skuId"] = ""
post_data[
"uuid"] = "16d14d36a98c8-0e53a36f17ba0e-1a201708-1fa400-16d14d36a98c8"
post_data["platform"] = "3"
post_data["partner"] = "4"
post_data[
"originUrl"] = "https://h5.waimai.meituan.com/waimai/mindex/menu?dpShopId=&mtShopId={}&utm_source=&source=shoplist&initialLat=23.129112&initialLng=113.264385&actualLat=&actualLng=".format(
poiId)
post_data["riskLevel"] = "71"
post_data["optimusCode"] = "10"
post_data["wm_latitude"] = "23129112"
post_data["wm_longitude"] = "113264385"
post_data["wm_actual_latitude"] = ""
post_data["wm_actual_longitude"] = ""
post_data[
"openh5_uuid"] = "16d14d36a98c8-0e53a36f17ba0e-1a201708-1fa400-16d14d36a98c8"
html = downloader.post("http://i.waimai.meituan.com/openh5/poi/food",
data=post_data).text
for key, val in ttf_replace.items():
html = html.replace(key, val)
json = json_util.to_python(html)
data = {
"shopName": json["shopInfo"]["shopName"],
"shopPic": json["shopInfo"]["shopPic"],
"deliveryFee": json["shopInfo"]["deliveryFee"],
"deliveryTime": json['data']['shopInfo']["deliveryTimeDecoded"],
}
time.sleep(3)