async def get_flag_text(self, data_url):
page = self._page_seller_flag
net_check()
while 1:
try:
await page.bringToFront()
await page.goto(data_url)
except errors.TimeoutError:
sleep(5)
except errors.PageError:
sleep(5)
else:
break
await asyncio.sleep(1)
content = await page.content()
await asyncio.sleep(2)
# await page.close()
await self.page.bringToFront()
doc = pq(content)
res = re.search('"tip":"(.*?)"}', doc("pre").text())
if res:
return res.group(1)
else:
logger.info(doc("pre").text())
return None
async def phone_verify(self, p):
try:
await p.waitForSelector("#container", timeout=120000)
except errors.TimeoutError:
logger.info("超时末扫码或需要手机验证！")
await self.verify(p)
net_check()
await p.goto("https://myseller.taobao.com/home.htm")
finally:
await p.waitForSelector("#container", timeout=0)
content = await p.content()
a = re.search('nick: "(.*?):', content)
b = re.search('nick: "(.*?)"', content)
if a:
account = a.group(1)
else:
account = b.group(1)
if account == "arduino_sz":
logger.info("开源电子登陆成功")
f = "KY"
elif account == "玉佳电子科技有限公司":
logger.info("玉佳企业店登陆成功")
f = "YK"
elif account == "simpleli":
logger.info("赛宝电子登陆成功")
f = "TB"
elif account == "selingna5555":
logger.info("玉佳电子登陆成功")
f = "YJ"
else:
logger.info('登陆账户信息获取失败，即将重启爬虫！')
await self.b.close()
await self.login()
try:
net_check()
await p.goto(
"https://trade.taobao.com/trade/itemlist/list_sold_items.htm"
)
await p.waitForSelector(
".pagination-mod__show-more-page-button___txdoB",
timeout=30000)
except errors.TimeoutError:
await self.verify(p)
net_check()
await p.goto(
"https://trade.taobao.com/trade/itemlist/list_sold_items.htm"
)
await p.waitForSelector(
".pagination-mod__show-more-page-button___txdoB",
timeout=30000)
finally:
net_check()
await p.click(".pagination-mod__show-more-page-button___txdoB"
) # 显示全部页码
t = await self.slider(p)
if t:
return t
else:
return f
async def verify(self, p):
try:
await p.waitForSelector("div.aq_overlay_mask", timeout=10000)
except errors.TimeoutError:
pass
else:
logger.info("需要要手机验证码")
await asyncio.sleep(10)
frames = p.frames
net_check()
await frames[1].click(".J_SendCodeBtn")
a = input(time_now() + " | 请输入6位数字验证码：")
await frames[1].type(".J_SafeCode", a,
{'delay': self.input_time_random() - 50})
net_check()
await frames[1].click("#J_FooterSubmitBtn")
async def run(self):
net_check()
await self.page.goto(self.url)
await asyncio.sleep(2)
await self.page.waitForSelector("input[name='queryItemId']", timeout=0)
frames = self.page.frames
frame = await self.get_nc_frame(frames)
if frame:
logger.info("条形验证码")
while True:
await asyncio.sleep(1)
await frame.hover("#nc_1_n1z")
await self.page.mouse.down()
await self.page.mouse.move(
2000, 0, {'delay': random.randint(1000, 2000)})
await self.page.mouse.up()
try:
frame.waitForSelector(".nc-lang-cnt a", timeout=10000)
await asyncio.sleep(2)
await frame.click(".nc-lang-cnt a")
except errors.TimeoutError:
await asyncio.sleep(1)
slider = await frame.J("#nc_1_n1z")
if not slider:
break
except errors.PageError:
await asyncio.sleep(1)
slider = await frame.J("#nc_1_n1z")
if not slider:
break
operator = ""
if MODE == 2:
operator = input(time_now() + " | 输入操作者名字:")
if operator:
self.operator = operator
logger.info("当前操作者 ：" + self.operator)
while True:
a = await self.fix_data()
if a == 1:
break
async def verify(self, p):
try:
await p.waitForSelector("div.aq_overlay_mask", timeout=10000)
except errors.TimeoutError:
pass
else:
logger.info("需要要手机验证码")
if LINUX:
test_server = ts.copy()
test_server['db'] = "test"
id = random.randint(0, 100)
mysql.insert_data(db=test_server,
t="phone_verify",
d={"id": id})
frames = p.frames
net_check()
verify_code = "0"
while True:
net_check()
await frames[1].click(".J_SendCodeBtn")
for i in range(120):
await asyncio.sleep(5)
res = mysql.get_data(
db=test_server,
cn=["verify_code"],
t="phone_verify",
c={"id": id},
)
verify_code = res[0][0]
if verify_code != "0":
mysql.delete_data(db=test_server,
t="phone_verify",
c={"id": id})
break
if verify_code != "0":
break
await asyncio.sleep(10)
else:
frames = p.frames
net_check()
await frames[1].click(".J_SendCodeBtn")
verify_code = input(time_now() + " | 请输入6位数字验证码：")
# await frames[1].click(".J_SendCodeBtn")
# verify_code = input(time_now() + " | 请输入6位数字验证码：")
await frames[1].type(".J_SafeCode", verify_code,
{'delay': self.input_time_random() - 50})
net_check()
await frames[1].click("#J_FooterSubmitBtn")
async def deliver(self, orderNo, shipNo, fromStore, shipCompany):
if fromStore == self.fromStore:
url = 'https://wuliu.taobao.com/user/consign.htm?trade_id=' + orderNo
page = await self.browser.newPage()
net_check()
await page.goto(url)
await page.waitForSelector(".ks-combobox-placeholder",
timeout=30000)
net_check()
await page.click("#offlineTab a")
await asyncio.sleep(1)
net_check()
await page.click(".font-blue.J_ChangeVision")
await asyncio.sleep(1)
print("offlineMailNo" + shipCompany)
await page.type("#offlineMailNo" + shipCompany, shipNo)
async def order_detail_spider(self, p, f):
sql1 = """
SELECT datailURL,a.orderNo FROM tb_order_spider a
JOIN taobaoorders b ON a.orderNo = b.OrderNo
WHERE isDetaildown=0 AND fromStore='%s' AND b.Flag = 8
ORDER BY createTime DESC;
""" % (f)
sql = """
SELECT datailURL,orderNo FROM tb_order_spider
WHERE isDetaildown=0 AND fromStore='%s'
ORDER BY createTime DESC
""" % (f)
results = mysql.get_data(sql=sql1, dict_result=True)
if not results:
results = mysql.get_data(sql=sql, dict_result=True)
if results:
for result in results:
order = {}
url = result['datailURL']
try:
net_check()
await p.goto(url)
except errors.TimeoutError:
continue
slider = await p.J('#nocaptcha')
if slider:
while True:
print("出现滑块验证码")
await asyncio.sleep(2)
await p.hover('#nc_1_n1z')
await p.mouse.down()
await p.mouse.move(
2000, 0, {'delay': random.randint(1000, 2000)})
await p.mouse.up()
try:
p.waitForSelector(".nc-lang-cnt a", timeout=10000)
await asyncio.sleep(2)
await p.click(".nc-lang-cnt a")
except errors.TimeoutError:
break
except errors.PageError:
break
try:
await p.waitForSelector('#detail-panel', timeout=30000)
except Exception as e:
continue
content = await p.content()
a = re.search("var data = JSON.parse\('(.*)'\);",
content).group(1)
b = a.replace('\\\\\\"', '')
data = b.replace('\\"', '"')
m = json.loads(data)
order['actualFee'] = m['mainOrder']['payInfo']['actualFee'][
'value']
order['orderStatus'] = status_format(
m['mainOrder']['statusInfo']['text'])
if order['orderStatus'] == '等待买家付款':
order['isDetaildown'] = 2
else:
order['isDetaildown'] = 1
coupon = 0
for k, v in m['mainOrder']['payInfo'].items():
if k == 'promotions':
promotions = m['mainOrder']['payInfo']['promotions']
for i in range(len(promotions)):
if 'prefix' and 'suffix' in promotions[i]:
coupon_temp = re.search(
"(\d+\.\d+)", promotions[i]['value'])
if coupon_temp:
coupon += float(coupon_temp.group(1))
order['couponPrice'] = round(coupon, 2)
for k, v in m.items():
if k == 'buyMessage':
order['buyerComments'] = v
orderNo = m['mainOrder']['id']
order_info = m['mainOrder']['orderInfo']['lines'][1]['content']
for i in range(len(order_info)):
if order_info[i]['value']['name'] == '支付宝交易号:':
try:
order['tradeNo'] = order_info[i]['value']['value']
except KeyError:
order['tradeNo'] = None
# elif order_info[i]['value']['name'] == '创建时间:':
# order['createTime'] = order_info[i]['value']['value']
# elif order_info[i]['value']['name'] == '发货时间:':
# order['shipTime'] = order_info[i]['value']['value']
elif order_info[i]['value']['name'] == '付款时间:':
order['payTime'] = order_info[i]['value']['value']
ship_info = m['tabs']
for i in range(len(ship_info)):
if ship_info[i]['id'] == 'logistics':
temp = ship_info[i]['content']
for k, v in temp.items():
if k == 'logisticsName':
order['shippingCompany'] = v
elif k == 'shipType':
order['shippingMethod'] = v
elif k == 'logisticsNum':
order['shippingNo'] = v
# elif k == 'logisticsUrl':
# order['shipUrl'] = "https" + v
elif k == 'address':
rec_info = v
order['receiverName'] = rec_info.split(
"，")[0].replace(" ", "")
order['receiverPhone'] = rec_info.split("，")[1]
order['receiverAddress'] = "".join(
rec_info.split("，")[2:])
sub_orders = m['mainOrder']['subOrders']
# mainOrder.subOrders[10].tradeStatus[0].content[0].value
line_no = 0
for i in range(len(sub_orders)):
if sub_orders[i]['tradeStatus'][0]['content'][0][
'value'] == '已取消':
continue
item = {}
temp = 0
itemNo = line_no
line_no += 1
if sub_orders[i]['promotionInfo']:
for j in sub_orders[i]['promotionInfo']:
for x in j['content']:
for k, v in x.items():
if k == 'value':
p_list = re.findall("-?\d+\.\d+", v)
if p_list:
temp += float(p_list.pop())
item['unitBenefits'] = temp
mysql.update_data(t="tb_order_detail_spider",
set=item,
c={
'orderNo': orderNo,
'itemNo': itemNo
})
mysql.update_data(t="tb_order_spider",
set=order,
c={'orderNo': orderNo})
Verify()
while True:
s = random.random()
if s > 0.9:
for i in range(int(s * n_o_time)):
await asyncio.sleep(1)
print(">", end="", flush=True)
print("")
break
else:
pass
await p.close()
await self.run_order_detail_spider()
async def login(self, page=None, **kwargs):
# shutil.rmtree(CHROME_PROFILE_PATH, True)
if not page:
page = await self.get_new_page()
while 1:
try:
net_check()
await page.goto(login_url)
except errors.PageError:
pass
except errors.TimeoutError:
pass
else:
break
while True:
try:
await page.waitForSelector(".forget-pwd.J_Quick2Static",
visible=True,
timeout=10000)
await page.click(".forget-pwd.J_Quick2Static")
except errors.TimeoutError:
pass
except errors.ElementHandleError:
await page.reload()
continue
finally:
try:
await page.type('#TPL_username_1', kwargs['username'],
{'delay': self.input_time_random() - 50})
await page.type('#TPL_password_1', kwargs['password'],
{'delay': self.input_time_random()})
except errors.ElementHandleError:
await page.reload()
else:
break
net_check()
# 检测页面是否有滑块。原理是检测页面元素。
slider = await page.Jeval('#nocaptcha', 'node => node.style') # 是否有滑块
if slider:
print("出现滑块情况判定")
await self.mouse_slide(p=page)
await page.click("#J_SubmitStatic") # 调用page模拟点击登录按钮。
time.sleep(2)
await self.get_cookie(page)
else:
await page.click("#J_SubmitStatic")
try:
await page.waitForSelector("#container", timeout=10000)
except errors.TimeoutError:
print("超时需要手机验证！")
frames = page.frames
try:
await frames[1].waitForSelector("button#J_GetCode",
timeout=10000)
except errors.TimeoutError:
pass
else:
print("需要要手机验证码")
test_server['db'] = "test"
id = random.randint(0, 100)
mysql.insert_data(db=test_server,
t="phone_verify",
d={"id": id})
# frames = page.frames
# await frames[1].click(".J_SendCodeBtn")
verify_code = "0"
while True:
net_check()
await frames[1].click("button#J_GetCode")
for i in range(120):
await asyncio.sleep(5)
res = mysql.get_data(
db=test_server,
cn=["verify_code"],
t="phone_verify",
c={"id": id},
)
verify_code = res[0][0]
if verify_code != "0":
mysql.delete_data(db=test_server,
t="phone_verify",
c={"id": id})
break
if verify_code != "0":
break
await frames[1].type("input#J_Phone_Checkcode", verify_code,
{"delay": self.input_time_random() - 50})
# await frames[1].type(".J_SafeCode", a, {'delay': self.input_time_random() - 50})
net_check()
await frames[1].click("input#submitBtn")
# await frames[1].click("#J_FooterSubmitBtn")
net_check()
await page.goto("https://myseller.taobao.com/home.htm")
await page.waitForSelector("#container", timeout=30000)
return self.b, page, kwargs['fromStore']
async def login(self, **kwargs):
p = await self.get_page(**kwargs)
while 1:
try:
await p.goto("https://login.taobao.com", timeout=30000)
except errors.PageError:
logger.warning("网络异常5秒后重连")
sleep(5)
except errors.TimeoutError:
logger.warning("网络异常5秒后重连")
sleep(5)
else:
break
while True:
try:
await p.waitForSelector(".forget-pwd.J_Quick2Static",
visible=True,
timeout=10000)
await p.click(".forget-pwd.J_Quick2Static")
except errors.TimeoutError:
pass
except errors.ElementHandleError:
await p.reload()
continue
finally:
try:
await p.type('#TPL_username_1', kwargs['username'],
{'delay': self.input_time_random() - 50})
await p.type('#TPL_password_1', kwargs['password'],
{'delay': self.input_time_random()})
except errors.ElementHandleError:
await p.reload()
else:
break
net_check()
# 检测页面是否有滑块。原理是检测页面元素。
try:
await p.waitForSelector('#nc_1_n1z', visible=True, timeout=3000)
except errors.TimeoutError:
slider = 0
else:
slider = await p.J('#nc_1_n1z')
if slider:
print("出现滑块情况判定")
t = await self.slider(p=p)
if t:
return self.b, p, t
await p.click("#J_SubmitStatic") # 调用page模拟点击登录按钮。
time.sleep(2)
await self.get_cookie(p)
else:
await p.click("#J_SubmitStatic")
try:
await p.waitForNavigation()
except errors.TimeoutError:
pass
print("登录成功")
return self.b, p
async def next_page(self, page_num=1):
"""执行翻页"""
temp = 0
while 1:
t = time_zone(["08:00", "18:00", "23:00"])
a = datetime.datetime.now()
if a < t[0]:
if not temp:
temp = 0
n_p_time = 600
elif t[0] < a < t[1]:
temp += 1
if temp == 1:
page_num = 1
n_p_time = NEXT_PAGE_TIME
elif a > t[2]:
n_p_time = 60
if not LINUX:
subprocess.call("shutdown /s")
exit("到点关机")
else:
n_p_time = 60
await self.page.bringToFront()
if self.orderno:
await self.page.focus("#bizOrderId")
await asyncio.sleep(1)
await self.page.keyboard.down("ShiftLeft")
await asyncio.sleep(1)
await self.page.keyboard.press("Home")
await asyncio.sleep(1)
await self.page.keyboard.down("ShiftLeft")
await asyncio.sleep(1)
await self.page.keyboard.press("Delete")
await asyncio.sleep(1)
orderno = input(time_now() + " | 输入订单号:")
await self.page.type("#bizOrderId", orderno)
await self.page.setRequestInterception(True)
self.page.on('request', self.intercept_request)
self.page.on('response', self.intercept_response)
net_check()
await self.page.click(".button-mod__primary___17-Uv")
await asyncio.sleep(10)
else:
while 1:
try:
await self.page.waitForSelector(
".pagination-options-go")
await self.page.focus(".pagination-options input")
# await self.page.click(".pagination-options input", clickCount=2)
await self.page.keyboard.press("Delete")
await self.page.keyboard.press("Delete")
await self.page.keyboard.press("Delete")
await self.page.keyboard.press("Backspace")
await self.page.keyboard.press("Backspace")
await self.page.keyboard.press("Backspace")
await self.page.setRequestInterception(True)
self.page.on('request', self.intercept_request)
self.page.on('response', self.intercept_response)
net_check()
await self.page.type(".pagination-options input",
str(page_num))
await self.page.keyboard.press("Enter")
self.page.waitForSelector(
".pagination-item.pagination-item-" +
str(page_num) + ".pagination-item-active",
timeout=10000)
except errors.TimeoutError:
logger.info('翻页超时，5秒后重新翻页')
sleep(5)
else:
break
# await self.page.waitForSelector(".pagination-item-" + str(page_num) + " a", timeout=30000)
# await self.page.click(".pagination-item-" + str(page_num) + " a")
while 1:
if self.complete == 1:
s = random.random()
if s > 0.5:
await self.link_spider()
await self.order_page()
logger.info(str(int(s * n_p_time)) + " 秒后开始下一页爬取")
sleep(int(s * n_p_time))
break
elif self.complete == 2:
page_num = 0
s = random.random()
if s > 0.9:
mysql.update_data(t="tb_order_spider",
set={"isDetaildown": 0},
c={
"isDetaildown": 2,
"fromStore": self.fromStore
})
sleep(int(s * n_p_time))
break
else:
# if i == 59:
# logger.info("超时")
# await self.page.screenshot({'path': './headless-test-result.png'})
await asyncio.sleep(3)
self.complete = 0
page_num += 1
async def parse(self, data):
if data != "q":
for i in range(len(data)):
self.item = {}
self.item = self.common.copy()
self.item['stockid'] = re.search(
"编码:(.*)",
data[i]['itemDesc']['desc'][1]['text']).group(1).upper()
self.item['link_id'] = data[i]['itemId']
self.item['attribute'] = ""
self.item['flag'] = "update"
self.item['typeabbrev'] = self.fromStore
self.item['shop_id'] = self.shop_id(self.fromStore)
self.item['SpiderDate'] = time_now()
temp_des = data[i]['itemDesc']['desc'][0]['text']
self.item['description'] = temp_des.replace("（", "(").replace(
"）", ")")
self.item['price_tb'] = re.findall(
"(\d+.?\d*)", data[i]["managerPrice"]['currentPrice'])[0]
self.item['promotionprice'] = self.promo_price.get(
self.item['link_id'])
# print(self.promo_price)
sql = "select spe_link from prices_tb_fix where link_id='%s' and server='%s'" % (
self.item['link_id'], self.sn)
spe_link_id = mysql.get_data(db=self.db_test,
sql=sql,
return_one=True)
isMut = re.search("^MUT\D*", self.item['stockid'])
if isMut or spe_link_id:
await self.page.setRequestInterception(True)
self.page.on('request', self.intercept_request)
self.page.on('response', self.intercept_response)
await asyncio.sleep(1)
net_check()
await self.page.click(
".next-table-row td:nth-child(2) div.product-desc-hasImg span:nth-child(2) i"
)
await asyncio.sleep(1)
await self.page.keyboard.press('Escape')
else:
# print(self.item)
if self.item['promotionprice'] is None:
mail("price_tb_error",
self.fromStore + ":" + self.item['link_id'],
["*****@*****.**"])
logger.error("error:" + self.fromStore + " : " +
self.item['link_id'] + " and " +
mysql.concat(self.promo_price, "="))
self.complete = 2
break
condition = {
"stockid": self.item['stockid'],
"link_id": self.item['link_id'],
"shop_id": self.item['shop_id'],
}
res = mysql.get_data(t="prices_tb",
l=1,
cn=["id"],
c=condition,
db=self.target_server)
if res:
self.item['ratio'] = round(
float(self.item['price_tb']) / float(res[0][0]), 2)
print(self.item)
mysql.update_data(t="prices_tb",
set=self.item,
c=condition,
db=self.target_server)
else:
insert_item = self.item.copy()
insert_item["currabrev"] = "CNY"
insert_item["price_erp"] = 0
insert_item["operator"] = self.operator
insert_item["last_time"] = time_now()
if self.operator == "爬虫维护":
insert_item["flag"] = "create"
else:
insert_item['flag'] = "add"
insert_item["ratio"] = 1
insert_item["package_number"] = 1
insert_item["Checker"] = ""
insert_item["CheckDate"] = "0000-00-00 00:00:00"
print(insert_item)
with open(
"reports/report_" + self.fromStore +
"_insert.txt", "a") as file:
file.writelines("物料编码：" + insert_item['stockid'] +
" 与 商品ID：" +
insert_item['link_id'] +
" 为最新匹配，添加至ERP系统。\n" +
self.item_url +
insert_item['link_id'] + "\n" +
self.item_erp_url +
insert_item['link_id'] + "\n\n")
mysql.insert_data(t="prices_tb",
d=insert_item,
db=self.target_server)
result = mysql.get_data(
t="prices_tb",
cn=["*"],
c={"link_id": self.item['link_id']},
db=self.target_server,
dict_result=True)
if len(result) > 1:
for r in result:
if r['stockid'] != self.item['stockid'] and r[
'flag'] != "del":
with open(
"reports/report_" + self.fromStore +
"_delete.txt", "a") as file:
file.writelines("物料编码：" + r['stockid'] +
" 与 商品ID：" +
self.item['link_id'] +
" 不匹配，已被爬虫从ERP系统中删除。\n" +
self.item_url +
self.item['link_id'] +
"\n" + self.item_erp_url +
self.item['link_id'] +
"\n\n")
mysql.update_data(t="prices_tb",
c={"id": r['id']},
db=self.target_server,
set={"flag": "del"})
self.complete = 1
else:
pass
self.complete = 1
async def get_page(self):
net_check()
await self.page.goto(self.url) # 跳转到页面
await self.next_page()
async def fix_data(self, link_id=None):
# page = await self.browser.newPage()
self.complete = 0
self.prices = {}
self.promo_price = {}
await asyncio.sleep(2)
await self.page.focus("input[name='queryItemId']")
await self.page.keyboard.down("ShiftLeft")
await self.page.keyboard.press("Home")
await self.page.keyboard.down("ShiftLeft")
await self.page.keyboard.press("Delete")
server_name = 'production_server'
self.sn = server_name
if not link_id:
if MODE == 1:
link_id = "585308692855"
elif MODE == 2:
while True:
link_id = input(time_now() + " | 输入link_id：")
isMatch = re.match("^\d{10,20}$", link_id)
if isMatch:
break
elif MODE == 3:
sql = """
SELECT link_id,updateTime,server,operator
FROM prices_tb_fix
WHERE fromStore='%s' and isComplete=0
ORDER BY flag LIMIT 1
""" % (self.fromStore)
res = mysql.get_data(sql=sql, db=self.db_test)
if res:
self.target_server = self.server[res[0][2]]
link_id = res[0][0]
updateTime = res[0][1]
server_name = res[0][2]
self.sn = server_name
self.operator = res[0][3]
else:
return 1
logger.info(link_id)
page = await self.browser.newPage()
await page.setViewport({'width': 1600, 'height': 900})
net_check()
await page.goto("https://item.taobao.com/item.htm?id=" + link_id,
timeout=0)
await asyncio.sleep(3)
error_page = await page.J(".error-notice-hd") # 判断宝贝是否正常在售
offline = await page.J("#J_detail_offline") # 判断宝贝是否正常在售
if error_page or offline:
logger.info("商品已下架")
mysql.update_data(t="prices_tb",
set={
"flag": "XiaJia",
"typeabbrev": self.fromStore
},
c={"link_id": link_id},
db=self.target_server)
# mysql.update_data(t="tb_order_detail_spider",
# set={"link_id": link_id + "xiajia"},
# c={"link_id": link_id},
# db=self.target_server)
mysql.update_data(db=self.db_test,
t="prices_tb_fix",
set={
"isComplete": "2",
"updateTime": time_now()
},
c={
"link_id": link_id,
"server": server_name
})
await page.close()
return
else:
while True:
content = await page.content()
# print(content)
doc = pq(content)
self.common['rates'] = doc.find("#J_RateCounter").text()
self.common['sales'] = doc.find("#J_SellCounter").text()
self.common['freight'] = doc.find("#J_WlServiceTitle").text()
mat1 = re.match("\d+", self.common['sales'])
mat2 = re.match("\d+", self.common['rates'])
if mat1 and mat2:
break
res = re.findall('";(.*?);".*?e":"(\d+\.\d+).*?d":"(\d+)"',
content) # 判断是否存在多属性
if res:
control = 1
benefit_price = 0
for r in res:
data_values = r[0].split(";")
prop = []
for data in data_values:
prop.append(
doc.find("li[data-value='" + data +
"'] span").text())
if control:
for data in data_values:
try:
await page.click('li[data-value="' + data +
'"]')
except errors.PageError:
pass
content_p = await page.content()
promo_price = re.findall(
'<em id="J_PromoPriceNum".*?>(\d+\.?\d*)</em>',
content_p) # 判断是否存在优惠
if len(promo_price) == 1:
benefit_price = float(r[1]) - float(promo_price[0])
control = 0
self.prices[r[2]] = r[1]
prop.reverse()
self.prop[r[2]] = "-".join(prop)
for r in res:
if benefit_price:
self.promo_price[r[2]] = round(
float(r[1]) - benefit_price, 2)
else:
promo_price = re.findall(
'<em id="J_PromoPriceNum".*?>(\d+.*\d*)</em>',
content) # 判断是否存在优惠
if promo_price:
self.promo_price[link_id] = promo_price[0]
else:
self.promo_price[link_id] = 0
# print(self.prices)
# print(self.promo_price)
await page.close()
await self.page.type("input[name='queryItemId']", link_id)
await self.page.setRequestInterception(True)
self.page.on('request', self.intercept_request)
self.page.on('response', self.intercept_response)
await asyncio.sleep(1)
net_check()
await self.page.click(".filter-footer button:first-child")
while True:
await asyncio.sleep(1)
if self.complete == 1:
res = mysql.get_data(db=self.db_test,
t="prices_tb_fix",
c={
"link_id": link_id,
"server": server_name
})
if res:
mysql.update_data(db=self.db_test,
t="prices_tb_fix",
set={
"isComplete": "1",
"updateTime": time_now()
},
c={
"link_id": link_id,
"server": server_name
})
break
elif self.complete == 2:
mysql.update_data(db=self.db_test,
t="prices_tb_fix",
set={"spe_link": "1"},
c={
"link_id": link_id,
"server": server_name
})
break
