async def run():
while 1:
update()
ms = MySql(db_setting=TEST_SERVER_DB_TEST)
ms.update(t="spider_monitor",
set={"latest_time": time_now()},
c={"spider_address": SPIDER_ADDRESS})
restart_signal = ms.get_one(t="spider_monitor",
cn=["restart_signal"],
c={"spider_address": SPIDER_ADDRESS})
if SPIDER_ADDRESS == "3_floor":
sql = "SELECT MAX(updateTime) as updateTime,fromStore FROM tb_order_spider WHERE fromStore IN ('KY','TB') GROUP BY fromStore"
else:
sql = "SELECT MAX(updateTime) as updateTime,fromStore FROM tb_order_spider WHERE fromStore IN ('YJ','YK') GROUP BY fromStore"
results = MySql.cls_get_dict(sql=sql)
t = time_ago(minutes=15)
for result in results:
if str(result['updateTime']) < t:
restart_signal = 1
break
if restart_signal:
ms.update(t="spider_monitor",
set={"restart_signal": 0},
c={"spider_address": SPIDER_ADDRESS})
restart()
del ms
await asyncio.sleep(60)
async def input_verify_code(self, frame, fromStore, type):
logger.info("需要要手机验证码")
ms = MySql(db_setting=TEST_SERVER_DB_TEST)
ms.delete(t='phone_verify', c={'fromStore': fromStore})
ms.insert(t="phone_verify", d={"fromStore": fromStore})
mail(fromStore + "手机验证码", fromStore + "登陆需要手机验证码", MAIL_RECEIVERS)
verify_code = "0"
while 1:
if type == 0:
await frame.click(PHONE_GET_CODE[0])
else:
await frame.click(PHONE_GET_CODE[1])
for i in range(120):
await asyncio.sleep(5)
verify_code = ms.get_one(t='phone_verify',
cn=['verify_code'],
c={"fromStore": fromStore})
if verify_code != "0":
ms.delete(t='phone_verify', c={'fromStore': fromStore})
del ms
break
if verify_code != "0":
break
await asyncio.sleep(10)
if type == 0:
await frame.type(PHONE_CHECK_INPUT[0], verify_code,
{'delay': self.input_time_random() - 50})
await frame.click(PHONE_SUBMIT_BTN[0])
else:
await frame.type(PHONE_CHECK_INPUT[1], verify_code,
{'delay': self.input_time_random() - 50})
await frame.click(PHONE_SUBMIT_BTN[1])
def get(self, shop_ids):
mail_container = self.init_receivers()
ms = MySql(db_setting=TEST_SERVER_DB_TEST)
for shop_id in shop_ids:
shop_name = MySql.cls_get_one(
sql="SELECT shopname FROM shop_info WHERE shop_id={}".format(
shop_id))
flag_report_groups = []
# sql = "SELECT COUNT(id) AS nums FROM tb_master where shop_id='{}' and update_date<'{}'".format(shop_id,
# datetime.date.today())
# nums = mysql.get_data(db=test_server, sql=sql, return_one=True)
# if int(nums) > 0:
# flag_report_groups.append("下架商品{}条".format(nums))
# del sql
# del nums
for flag in self.translate_dictionary.keys():
sql = "SELECT COUNT(id) AS nums FROM tb_master WHERE flag LIKE '%%{}%%' AND shop_id='{}'".format(
flag, shop_id)
nums = ms.get_one(db=TEST_SERVER_DB_TEST, sql=sql)
if int(nums) > 0:
flag_report_groups.append("{}{}条".format(
self.translate_dictionary[flag], nums))
if flag_report_groups:
result = ms.get_dict(t="tb_search_page_info",
c={"shop_id": shop_id})
flag_report_groups.append("总计爬取{}页".format(
result[0]['total_page']))
flag_report_groups.append("总计花费{}分{}秒".format(
int(result[0]['spent_time'] / 60),
int(result[0]['spent_time'] % 60)))
flag_report_groups.reverse()
flag_report_groups.append(shop_name)
flag_report_groups.reverse()
if shop_id in ["115443253", "33817767"]:
mail_container["KY"]['mail_content'] += "|".join(
flag_report_groups) + "\n"
mail_container["KY"]['mail_content'] += self.insert_link(
shop_id, ms)
elif shop_id in ["34933991", "131282813"]:
mail_container["TB"]['mail_content'] += "|".join(
flag_report_groups) + "\n"
mail_container["TB"]['mail_content'] += self.insert_link(
shop_id, ms)
elif shop_id in ["68559944", "60299985"]:
mail_container["YJ"]['mail_content'] += "|".join(
flag_report_groups) + "\n"
mail_container["YJ"]['mail_content'] += self.insert_link(
shop_id, ms)
else:
mail_container["YK"]['mail_content'] += "|".join(
flag_report_groups) + "\n"
mail_container["YK"]['mail_content'] += self.insert_link(
shop_id, ms)
return mail_container
async def do_it(self):
shop_id = store_trans(self.fromStore, 'code_2_id')
ms = MySql()
sql = "select link_id from prices_tb where need_to_update=1 and shop_id='{}' limit 1".format(
shop_id)
link_id = ms.get_one(sql=sql)
if not link_id:
return 0
await self.page.bringToFront()
try:
if not re.search(
"https://item.manager.taobao.com/taobao/manager/render.htm",
self.page.url):
await self.page.goto(
"https://item.manager.taobao.com/taobao/manager/render.htm?tab=on_sale"
)
except Exception as e:
logger.error(str(e) + "manager_page_error")
return
while 1:
await self.page.waitForSelector("input[name='queryItemId']",
timeout=0)
await self.page.keyboard.press('Escape')
await self.page.focus("input[name='queryItemId']")
for _ in range(20):
await self.page.keyboard.press("Delete")
await self.page.keyboard.press("Backspace")
await self.page.type("input[name='queryItemId']", str(link_id),
{'delay': self.login.input_time_random()})
await self.page.click(".filter-footer button:first-child")
await self.page.waitForResponse(
"https://item.manager.taobao.com/taobao/manager/table.htm")
await asyncio.sleep(1)
await self.listening(self.page)
try:
await self.page.waitForSelector(FAST_EDIT_BTN, timout=10000)
await self.page.click(FAST_EDIT_BTN)
restart = await self.login.slider(self.page)
if restart:
exit("滑块验证码失败，退出")
except errors.TimeoutError as e:
logger.info("商品已下架，没有查询到对应的商品ID：" + link_id)
ms.update(t="prices_tb",
set={
"SpiderDate": time_now(),
"need_to_update": 0,
"flag": "XiaJia"
},
c={"link_id": link_id})
link_id = ms.get_one(sql=sql)
if not link_id:
return 0
continue
else:
await self.page.focus("input[name='queryItemId']")
for _ in range(20):
await self.page.keyboard.press("Delete")
await self.page.keyboard.press("Backspace")
break
while 1:
if self.completed == 4:
break
await asyncio.sleep(1)
await asyncio.sleep(15)