def get_baidu_pages(self, page):
try:
baidu_p = WebDriverWait(self.browser, 5).until(
EC.presence_of_element_located(
(By.XPATH,
f"//span[@class='pc' and contains(text(),'{page}')]"
))).find_element_by_xpath('..')
return baidu_p
except Exception as e:
logrecord.log("获取百度前指定页面节点异常" + e)
raise Exception("获取百度前指定页面节点异常" + e)
def get_baidu_kz_url(self):
try:
kz_list = []
baidu_kz = WebDriverWait(self.browser, 5).until(
EC.presence_of_all_elements_located(
(By.CSS_SELECTOR, 'div.se_st_footer>a.m')))
for item in baidu_kz:
href = item.get_attribute('href')
kz_list.append(href)
return kz_list
except Exception as e:
logrecord.log("获取快照链接信息异常" + e)
raise Exception("获取百度前指定页面节点异常" + e)
def run(urls):
logrecord.log(f'检索的快照页为:{urls}')
phones_list=[]
res = baidukz_download(urls)
for item in res:
if 'html' in item:
html = item['html']
phones_temp = getReMatchPhones(html)
source_url = getSourceDomain(html)
if not source_url:
source_url = item['url']
kz_url = item['url']
phones_list.append({'phones': phones_temp, 'source_url': source_url, 'kz_url': kz_url})
return phones_list
def insert_phone_sms_logs(self, phone, keyword, content, batchno, respcode,
respdesc, logid):
try:
logrecord.log(f"短信发送记录添加到数据库中")
self.session.add(
PhoneSmsLogs(phone=phone,
keyword=keyword,
content=content,
batchno=batchno,
respcode=respcode,
respdesc=respdesc,
logid=logid,
create_time=int(time.time())))
self.session.commit()
except Exception as e:
logrecord.log(f"短信发送记录添加到数据库异常:{str(e)}")
self.session.rollback()
def get_all_kz_urls(self):
kz_list = []
try:
#获取第一页内容
kz_temp = self.get_baidu_kz_url()
kz_list = kz_list + kz_temp
for page in range(2, 11, 1):
sleep(3)
page_el = self.get_baidu_pages(page)
if page_el:
page_el.click()
kz_temp = self.get_baidu_kz_url()
kz_list = kz_list + kz_temp
return kz_list
except Exception as e:
logrecord.log("百度调用异常" + e)
finally:
return kz_list
def import_excel_file(self):
pageindex = 1
pagesize = 1000
flag = True
dbservice = DbService()
book = Workbook()
sheet = book.active
sheet.append(["手机号", "关键词", "发送内容", "回执状态", "回执描述", "发送时间"])
try:
while flag:
res = dbservice.get_export_phone_sms_logs_data(
pagesize=pagesize, pageindex=pageindex)
if (res):
for i, row in enumerate(res):
logrecord.log(row)
sheet.append(row)
else:
flag = False
pageindex += 1
book.save(self.fname)
except Exception:
traceback.print_exc()
finally:
self.sigexportphonesmslog.emit()
def checkDataSign(data):
buff = []
for k in data:
if k != "sign" and data[k] != "" and not (isinstance(data[k], dict) or
isinstance(data[k], list)):
buff.append(f'{k}={data[k]}')
sbuff = sorted(buff)
tmpStr = '&'.join(sbuff)
stringSignTemp = f'{tmpStr}&key={app.MD5_KEY}'
logrecord.log(f"签名的数据包:{stringSignTemp}")
sign = rsa.EnMd5(stringSignTemp).upper()
logrecord.log(f"验证的签名包:{sign}")
logrecord.log(f"接口返回的签名包:{data['sign']}")
return data['sign'] == sign
def set_phones_to_db(self, phones, source_url, kz_url, keyword):
phones_list = phones
db_phones_list = []
#域名去重复
no_repeat_phones_list = list(set(phones_list))
# 保持顺序一致
no_repeat_phones_list.sort(key=phones_list.index)
#判断有多少存在数据库中
if len(no_repeat_phones_list) > 0:
res_phones = self.session.query(Phones).filter(
Phones.phone.in_(no_repeat_phones_list)).all()
for dbitem in res_phones:
db_phones_list.append(dbitem.phone)
logrecord.log(f"重复手机号:{str(db_phones_list)},不记录到数据库")
#求出数据库没有的值
no_db_list_phones = list(
set(no_repeat_phones_list).difference(set(db_phones_list)))
insert_data = []
for item in no_db_list_phones:
insert_data.append({
"phone": str(item),
"keyword": str(keyword),
"source_url": str(source_url),
"kz_url": str(kz_url),
"create_time": int(time.time())
})
if len(insert_data) > 0:
logrecord.log(f"新手机号:{str(no_db_list_phones)},记录到数据库中")
try:
self.session.execute(Phones.__table__.insert(), insert_data)
self.session.commit()
except Exception as e:
logrecord.log(f"数据库异常:{str(e)}")
self.session.rollback()