def _get_captcha_code_by_chrome(self, driver, username):
captcha_xpath = "//img[@id='rndCodeCue']"
self.wait_xpath(driver, captcha_xpath)
captcha_element = driver.find_element_by_xpath(captcha_xpath)
location = captcha_element.location
size = captcha_element.size
left = location["x"]
top = location["y"]
photo_base64 = driver.get_screenshot_as_base64()
captcha_body = driver_screenshot_2_bytes(
photo_base64,
(left, top, left + size["width"], top + size["height"]))
return self.ask_image_captcha(captcha_body, username, file_type=".png")
def __get_captcha_code(self, driver, username, flag=True):
if flag:
driver.switch_to.frame('ICBC_login_frame')
else:
driver.switch_to.frame('integratemainFrame')
driver.switch_to.frame('VerifyimageFrame')
captcha_xpath = "//img[@title='点击图片可刷新']"
self.wait_xpath(driver, captcha_xpath)
location = driver.find_element_by_xpath(
captcha_xpath).location_once_scrolled_into_view
left = location["x"] + 8
top = location["y"]
driver.switch_to.default_content()
photo_base64 = driver.get_screenshot_as_base64()
captcha_body = driver_screenshot_2_bytes(
photo_base64, (left, top, left + 90, top + 36))
return self.ask_image_captcha(captcha_body, username, file_type=".png")
def login(self, driver, wait, response):
meta = response.meta
try:
main_div_xpath = "//form[@id='login_form2']"
driver.get(response.url)
# 识别验证码
wait.until(lambda dr: dr.find_element_by_xpath(
main_div_xpath + "//div[@class='qrcode']/img").is_displayed())
captcha_ele = driver.find_element_by_xpath(
main_div_xpath + "//div[@class='qrcode']/img")
location = captcha_ele.location
size = captcha_ele.size # 获取验证码的长宽
left, top = location['x'], location['y']
photo_base64 = driver.get_screenshot_as_base64()
temp = driver_screenshot_2_bytes(
photo_base64,
(left, top, left + size['width'], top + size['height']))
captcha_code = recognize_captcha_auto(temp,
digits_only=True,
del_noise=True)
if len(captcha_code) != 3:
raise CaptcherInvalidException()
_cookies = get_cookies_dict_from_webdriver(driver)
# 填写input
input_capimg = driver.find_element_by_xpath(
main_div_xpath + "//input[@type='hidden'][1]")
input_cookiename = driver.find_element_by_xpath(
main_div_xpath + "//input[@type='hidden'][2]")
input_username = driver.find_element_by_xpath(
main_div_xpath + "//input[@type='text'][1]")
input_password = driver.find_element_by_xpath(
main_div_xpath + "//input[@type='password'][1]")
input_qrcode = driver.find_element_by_xpath(
main_div_xpath + "//div[@class='qrcode']/input")
item = meta["item"]
formdata = {
input_capimg.get_attribute("name"):
input_capimg.get_attribute("value"),
input_cookiename.get_attribute("name"):
input_cookiename.get_attribute("value"),
input_username.get_attribute("name"):
item["username"],
input_password.get_attribute("name"):
item["password"],
input_qrcode.get_attribute("name"):
captcha_code
}
return FormRequest(
"http://jypt.cdhrss.gov.cn:8048/mlogin_action.php?id=1",
meta=meta,
cookies=_cookies,
formdata=formdata,
dont_filter=True,
callback=self.login_step2,
errback=self.err_callback)
except CaptcherInvalidException:
raise
except Exception:
raise LoginException(response.url)
def parse(self, response):
meta = response.meta
item = meta['item']
username = item["username"]
driver = self.load_page_by_webdriver(self._start_url_)
sleep(0.5)
try:
try:
driver.execute_script("""$('input[ng-model="account"]').val('%s');
$('input[ng-model="account"]').change();""" % username)
sleep(0.2)
driver.execute_script("""$('input[ng-model="pwd"]').val('%s');
$('input[ng-model="pwd"]').change();""" % item["password"])
sleep(0.2)
driver.execute_script("document.getElementsByClassName('btn-login fontFamily')[0].click();")
sleep(2)
if driver.current_url == self._start_url_:
captcha_image = driver.find_element_by_xpath('//div[@ng-show="needCaptcha"]/img')
location = captcha_image.location
size = captcha_image.size
left = location["x"] - 8
top = location["y"]
right = left + size["width"]
bottom = top + size["height"]
photo_base64 = driver.get_screenshot_as_base64()
captcha_body = driver_screenshot_2_bytes(photo_base64, (left, top, right, bottom))
captcha_code = self.ask_image_captcha(captcha_body, username, file_type=".png")
driver.execute_script("""$('input[ng-model="captcha"]').val('%s');
$('input[ng-model="captcha"]').change();""" % captcha_code)
driver.execute_script("document.getElementsByClassName('btn-login fontFamily')[0].click()")
# 登录成功之后 直接访问json页面, 提取对应数据进行筛选
# sleep(0.5)
except TypeError:
pass
except AttributeError:
# 跳转页面会出现异常,这个异常可忽略.
pass
driver.get(self.search_url % (0, get_js_time(), self.keyword))
result_str = self.mail_list_pattern.search(driver.page_source).group(1)
result = json_loads(result_str)
if result['msg'] != '登录超时，请重新登录':
bill_records = item["bill_records"]
the_data = result['data']
bill_records.extend(self._get_bill_records(driver, the_data['list']))
page_step = self.page_step
page = (the_data['total'] + page_step - 1) // page_step
for pa in range(1, page): # 过滤掉第一页
driver.get(self.search_url % (pa * page_step, get_js_time(), self.keyword))
result_str = self.mail_list_pattern.search(driver.page_source).group(1)
bill_records.extend(self._get_bill_records(driver, json_loads(result_str)['data']['list']))
yield from self.crawling_done(item)
else:
yield from self.error_handle(username, "搜狐邮箱---等待验证码超时",
tell_msg="等待验证码超时，请刷新页面重试。。")
except Exception:
yield from self.except_handle(username, "搜狐邮箱---抓取异常",
tell_msg="请刷新页面重试。。")
finally:
driver.quit()
def parse(self, response):
meta = response.meta
item = meta["item"]
username = item["username"]
driver = self.getdriver(
executable_path=self.settings["CHROME_EXECUTABLE_PATH"],
browser_type="CHROME_HEADLESS")
try:
wait = ui.WebDriverWait(driver, 10)
driver.get(response.url)
# 输入用户名
username_input = wait.until(
EC.visibility_of_element_located(
(By.ID, 'txt_username_79443')))
username_input.send_keys(username)
username_input.send_keys(Keys.TAB)
# 输入密码
try:
pwd_input = wait.until(
EC.visibility_of_element_located(
(By.ID, 'input_div_password_79445_1')))
except TimeoutException:
pwd_input = wait.until(
EC.visibility_of_element_located(
(By.ID, 'input_div_password_79445')))
pwd_input.send_keys(item["password"])
self.click_hidemsgbox(wait)
# 输入验证码
try:
try:
validation_img = wait.until(
EC.visibility_of_element_located(
(By.ID, 'captcha_debitCard')))
except TimeoutException:
validation_img = wait.until(
EC.visibility_of_element_located((By.ID, 'captcha')))
left = validation_img.location['x']
top = validation_img.location['y']
right = validation_img.location['x'] + validation_img.size[
'width']
bottom = validation_img.location['y'] + validation_img.size[
'height']
photo_base64 = driver.get_screenshot_as_base64()
img_bytes = driver_screenshot_2_bytes(
photo_base64, (left, top, right, bottom))
captcha_code = self.ask_image_captcha(img_bytes,
username,
file_type=".png")
self.click_hidemsgbox(wait)
try:
validation_input = wait.until(
EC.visibility_of_element_located(
(By.ID, 'txt_captcha_741012')))
except TimeoutException:
validation_input = wait.until(
EC.visibility_of_element_located(
(By.ID, 'txt_captcha_79449')))
validation_input.send_keys(captcha_code)
self.click_hidemsgbox(wait)
except Exception:
pass
# 点击登录
try:
submit_input = wait.until(
EC.visibility_of_element_located((By.ID, 'btn_49_741014')))
except TimeoutException:
submit_input = wait.until(
EC.visibility_of_element_located(
(By.ID, 'btn_login_79676')))
submit_input.click()
# 判断是否登录成功
try:
error_info = wait.until(
EC.visibility_of_element_located((By.ID, 'msgContent')))
msg = error_info.text
if msg != "":
yield from self.error_handle(username, msg, tell_msg=msg)
return
except TimeoutException:
pass
if username.isdigit() and len(username) == 19:
# 使用webdriver的方式
# 查询余额
balance_td = wait.until(
EC.visibility_of_element_located(
(By.XPATH, '//*[@id="cardMain"]/div/div[2]'
'/table/tbody/tr/td[4]')))
item["balance"] = balance_td.text.strip()
# 选择查询交易明细
trade_detail = wait.until(
EC.visibility_of_element_located(
(By.ID, 'div_transaction_details_740993')))
trade_detail.click()
# 选择日期
start_date = wait.until(
EC.visibility_of_element_located(
(By.ID, 'date_start_date_740972')))
begin_date = (date.today() -
timedelta(days=180)).strftime('%Y/%m/%d')
start_date.clear()
start_date.send_keys(begin_date)
# 点击空白
null_click = wait.until(
EC.visibility_of_element_located(
(By.ID, 'sel_pleaseselectaccount_740896')))
null_click.click()
# 点击查询按钮
query_input = wait.until(
EC.visibility_of_element_located((By.ID, 'btn_49_740974')))
query_input.click()
trade_records = item["trade_records"]
# 获取总页码
page_count_em = driver.find_element_by_xpath(
'//*[@id="pager"]/ul/li[2]/em[2]')
page_count = int(page_count_em.text)
next_count = 0
while next_count < page_count:
# 滚到顶部
driver.execute_script("window.scrollTo(0,0)")
wait.until(
EC.visibility_of_element_located(
(By.XPATH, '//*[@id="debitCardTransDetail_table"]'
'/table/tbody/tr')))
tr_list = driver.find_elements_by_xpath(
'//*[@id="debitCardTransDetail_table"]/table/tbody/tr')
for tr in tr_list:
td = tr.find_elements_by_xpath("td")
trade_income = td[6].text
trade_outcome = td[7].text
tmp_dict = {
"trade_date": td[0].text,
"trade_remark": td[1].text,
"trade_acceptor_account": td[2].text,
"trade_acceptor_name": td[3].text,
"trade_currency": td[4].text,
"trade_income": trade_income,
"trade_outcome": trade_outcome,
"trade_amount": trade_income
or ("-" + trade_outcome),
"trade_balance": td[8].text,
"trade_channel": td[9].text, # 交易渠道
}
trade_records.append(tmp_dict)
next_count += 1
# 滚到页面底部才有下一页按钮
driver.execute_script(
"window.scrollTo(0,document.body.scrollHeight)")
next_pager = wait.until(
EC.visibility_of_element_located(
(By.CLASS_NAME, 'next')))
next_pager.click()
sleep(1)
driver.execute_script("window.scrollTo(0,0)")
# 退出网银
pager_quit = wait.until(
EC.visibility_of_element_located((By.ID, 'a_59_1168')))
pager_quit.click()
# 抓取完成
yield from self.crawling_done(item)
else:
cookies = driver.get_cookies()
# 点击银行账户链接
my_account = wait.until(
EC.visibility_of_element_located(
(By.XPATH, '//a[@key="MyAccount"]')))
my_account.click()
my_balance = wait.until(
EC.visibility_of_element_located(
(By.XPATH, '//a[@title="余额"]')))
# 获取accountid
accountid = my_balance.get_attribute('accountid')
my_balance.click()
# 获取余额
item["balance"] = wait.until(
EC.visibility_of_element_located(
(By.XPATH, '//td[@lan="num"]'))).text
begin_date = (date.today() -
timedelta(days=180)).strftime('%Y/%m/%d')
end_date = date.today().strftime('%Y/%m/%d')
# 下载账单
download_url = "%s?accountId=%s¤cy=%s&cashRemit=&startDate=%s&endDate=%s" \
% (self.download_url, accountid, "001", begin_date, end_date)
yield Request(url=download_url,
callback=self.parse_download,
headers=self.headers,
meta=meta,
cookies=cookies,
dont_filter=True,
errback=self.err_callback)
except CaptchaTimeout:
yield from self.error_handle(username,
"中国银行---等待验证码超时",
tell_msg="等待验证码超时，请刷新页面重试。。")
except Exception:
yield from self.except_handle(username, "中国银行---数据解析异常")
finally:
driver.quit()
def parse(self, response):
item = response.meta["item"]
username = item["username"]
driver = self.getdriver(
executable_path=self.settings["IE_EXECUTABLE_233_PATH"],
browser_type="IE")
try:
wait = ui.WebDriverWait(driver, 20)
driver.get(response.url)
wait.until(lambda dr: dr.find_element_by_id("loginId"))
driver.maximize_window()
pid = driver.iedriver.process.pid
with Ddxoft(pid) as visual_keyboard:
driver.execute_script(
'user_input=document.getElementById("loginId");'
'user_input.focus();'
'user_input.value="' + username + '";')
visual_keyboard.dd_tab()
sleep(0.1)
for key in item["password"]:
visual_keyboard.dd_keyboard(key)
sleep(0.1)
# 检查是否需要输入验证码
captcha_input = driver.find_element_by_id("captcha")
if captcha_input and captcha_input.is_displayed():
captcha_image = driver.find_element_by_id("verifyImg")
location = captcha_image.location
size = captcha_image.size
left = location["x"] - 5
top = location["y"]
right = left + size["width"]
bottom = top + size["height"]
photo_base64 = driver.get_screenshot_as_base64()
captcha_body = driver_screenshot_2_bytes(
photo_base64, (left, top, right, bottom))
captcha_code = self.ask_image_captcha(captcha_body,
username,
file_type=".png")
captcha_input.send_keys(captcha_code)
driver.execute_script(
'document.getElementById("captcha").value="' +
captcha_code + '";')
driver.execute_script(
'document.getElementById("loginButton").click();')
wait.until(lambda dr: dr.find_element_by_id("_b01").is_displayed())
driver.execute_script(
'document.getElementById("_b01").children[1].style.display="block";'
'document.getElementById("_b01").children[1].style.left="-49px";'
)
sleep(5)
# with open("webpage.html", "w", encoding="utf-8") as file
# file.write(driver.page_source)
# 跳转到银行流水界面
trade_detail_btn = driver.find_element_by_xpath(
"//a[contains(text(),'交易明细')]")
if trade_detail_btn:
trade_detail_btn.click()
wait.until(lambda dr: dr.find_element_by_id("windowContainer").
is_displayed())
driver.switch_to.frame(
driver.find_element_by_id(
"windowContainer").find_element_by_xpath("//iframe"))
start_date_input = driver.find_element_by_id("beginDate")
end_date_input = driver.find_element_by_id("endDate")
submit_input = driver.find_element_by_id("advancedQueryTable")\
.find_element_by_xpath("//a[contains(text(),'查询')]")
end_date = date.today()
trade_detail = []
for i in range(4):
start_date = self.get_theday_3_month_ago(end_date)
try:
start_date_input.clear()
start_date_input.send_keys(
start_date.strftime("%Y-%m-%d"))
end_date_input.clear()
end_date_input.send_keys(end_date.strftime("%Y-%m-%d"))
submit_input.click()
wait.until(lambda dr: dr.find_element_by_id(
"advancedQueryTable").is_displayed())
trade_detail.extend(
self.parse_item_from_webpage(driver, wait, item))
except Exception:
pass
end_date = start_date
item["trade_records"] = trade_detail
yield from self.crawling_done(item)
else:
yield from self.error_handle(username,
msg="广发银行---错误信息：在页面中找不到[交易明细]按钮",
tell_msg="银行流水数据爬取失败，请刷新页面重试!")
except CaptchaTimeout:
yield from self.error_handle(username,
"广发银行---等待验证码超时",
tell_msg="等待验证码超时，请刷新页面重试。。")
except Exception:
yield from self.except_handle(username,
msg="广发银行---错误信息：",
tell_msg="银行流水数据爬取失败，请刷新页面重试!")
finally:
driver.quit()
def parse(self, response):
meta = response.meta
item = meta['item']
username = item['username']
password = item['password']
driver = self.load_page_by_webdriver(response.url)
try:
driver.implicitly_wait(1)
driver.execute_script(
'document.getElementsByClassName("menu_item")[0].click();')
driver.implicitly_wait(1)
driver.execute_script(
'document.getElementsByName("logonId")[0].value="%s";'
'document.getElementsByName("getSms")[0].click();' % username)
page = str(bs_format(driver.page_source, "lxml"))
if "短信已发送" in page:
self.logger.info("--->短信已发送")
sms_code = self.ask_sms_captcha(username)
self.logger.info("短信验证码: %s" % sms_code)
# 输入验证码
validation_img = driver.find_element_by_xpath(
'.//*[@alt="checkCode"]')
left = validation_img.location['x']
top = validation_img.location['y']
right = validation_img.location['x'] + validation_img.size[
'width']
bottom = validation_img.location['y'] + validation_img.size[
'height']
photo_base64 = driver.get_screenshot_as_base64()
img_bytes = driver_screenshot_2_bytes(
photo_base64, (left, top, right, bottom))
captcha_code = self.ask_image_captcha(img_bytes,
username,
file_type=".png")
self.logger.info("图片验证码: %s" % captcha_code)
driver.execute_script(
'document.getElementsByName("smsPassword")[0].value="%s";'
'document.getElementsByName("password")[0].value="%s";'
'document.getElementsByName("checkCode")[0].value="%s";'
'document.getElementsByName("login")[0].click();' %
(sms_code, password, captcha_code))
driver.implicitly_wait(1)
res_page = str(bs_format(driver.page_source, "lxml"))
if "登录信息输入有误" in res_page:
yield from self.error_handle(
username,
msg="广发银行---登录信息输入有误",
tell_msg="银行流水数据爬取失败，请刷新页面重试!")
elif "accmanage" in res_page:
self.logger.info("登录成功")
cookies = driver.get_cookies()
tran_his_url = "https://wap.cgbchina.com.cn/tranHistoryQueryInput.do?currentMenuId=101002"
yield Request(tran_his_url,
callback=self.parse_balance_info,
meta=meta,
headers=self.headers,
cookies=cookies,
dont_filter=True,
errback=self.err_callback)
else:
yield from self.error_handle(
username,
msg="广发银行---未知错误",
tell_msg="银行流水数据爬取失败，请刷新页面重试!")
else:
yield from self.error_handle(username,
msg="广发银行---短信发送失败",
tell_msg="银行流水数据爬取失败，请刷新页面重试!")
except CaptchaTimeout:
yield from self.error_handle(username,
"广发银行---等待验证码超时",
tell_msg="等待验证码超时，请刷新页面重试。。")
except Exception:
yield from self.except_handle(username,
msg="广发银行---登录失败",
tell_msg="银行流水数据爬取失败，请刷新页面重试!")
finally:
driver.quit()
def parse(self, response):
meta = response.meta
item = meta["item"]
user_name = item['username']
driver = self.load_page_by_webdriver(response.url)
try:
driver.maximize_window()
# username_input = driver.find_element_by_id('alias')
# username_input.send_keys(user_name)
pid = driver.iedriver.process.pid
with Ddxoft(pid) as dd_opt:
driver.execute_script(
'document.getElementById("alias").value="{0}";'
'document.getElementById("alias").focus()'.format(
user_name))
sleep(0.5)
dd_opt.dd_tab()
sleep(0.5)
for i in item['password']:
dd_opt.dd_keyboard(i)
try:
captcha_img = driver.find_element_by_class_name(
'captchas-img-bg')
input_captcha = driver.find_element_by_id('input_captcha')
if captcha_img.is_displayed():
photo_base64 = driver.get_screenshot_as_base64()
left = captcha_img.location['x'] - 8
top = captcha_img.location['y']
right = captcha_img.location['x'] + captcha_img.size[
'width'] - 8
bottom = captcha_img.location['y'] + captcha_img.size[
'height']
captcha_body = driver_screenshot_2_bytes(
photo_base64, (left, top, right, bottom))
captcha_code = self.ask_image_captcha(captcha_body,
user_name,
file_type=".png")
# driver.execute_script('document.getElementById("input_captcha").value="{0}";'
# .format(captcha_code))
input_captcha.send_keys(captcha_code)
except NoSuchElementException:
pass
butt_submit = driver.find_element_by_id('login')
self.element_click_three_times(butt_submit)
sleep(3)
try:
driver.find_element_by_id('login')
yield from self.error_handle(
user_name,
"交通银行---登录失败：(username:%s, password:%s) %s" %
(user_name, item["password"], '账号、密码错误或模拟键盘操作失败,返回登录页面'),
tell_msg="账号、密码错误或模拟键盘操作失败")
return
except NoSuchElementException:
pass
try:
# 有可能没有验证手机
sleep(1)
btnSendCode = driver.find_element_by_id('authSMSSendBtn')
self.element_click_three_times(btnSendCode)
sms_code = self.ask_sms_captcha(user_name)
sms_code_input = driver.find_element_by_id('mobileCode')
sms_code_input.send_keys(sms_code)
sleep(0.5)
submit = driver.find_element_by_id('btnConf2')
self.element_click_three_times(submit)
sleep(1)
except NoSuchElementException:
pass
# 提交设置为常用电脑
try:
self.element_click_three_times(
driver.find_element_by_xpath(
'//input[@checked="checked"]'))
self.element_click_three_times(
driver.find_element_by_id('next'))
sleep(0.5)
except NoSuchElementException:
pass
self.wait_xpath(driver, '//iframe[@id="frameMain"]')
driver.switch_to.frame('frameMain')
driver.execute_script(
'document.getElementById("search").value="账户明细查询";')
# driver.find_element_by_id('search').send_keys('账户明细查询')
self.element_click_three_times(
driver.find_element_by_class_name('search_btn'))
self.wait_xpath(driver, '//iframe[@id="tranArea"]')
frame = driver.find_element_by_id('tranArea')
# 转到 主工作区 iframe 里面
driver.switch_to.frame(frame)
# 搜索框中的交易明细查询 menucode为P002000
if driver.find_elements_by_xpath('//td[@menucode="P002000"]/a'):
self.element_click_three_times(
driver.find_element_by_xpath(
'//td[@menucode="P002000"]/a'))
sleep(1)
fromday = (date.today() - self.date_delta).strftime("%Y%m%d")
driver.execute_script(
'$("#startDate").datepicker("setDate","{0}");'.format(
fromday))
submit_date = driver.find_element_by_id('btnQry2')
self.element_click_three_times(submit_date)
sleep(0.5)
self.wait_xpath(driver, '//tbody[@id="recordtbody"]')
page_source = driver.page_source
result = self.__get_page_detail(page_source)
for i in range(100):
try:
pageDown_btn = driver.find_element_by_id('pageDown')
except NoSuchElementException:
break
if pageDown_btn.is_displayed():
self.element_click_three_times(pageDown_btn)
sleep(1)
page_source = driver.page_source
result.extend(self.__get_page_detail(page_source))
else:
break
item['trade_records'] = result
yield from self.crawling_done(item)
else:
yield from self.error_handle(user_name,
"交通银行---未搜索到交易明细",
tell_msg="信息获取异常")
except CaptchaTimeout:
yield from self.error_handle(user_name,
"交通银行---等待验证码超时",
tell_msg="等待验证码超时，请刷新页面重试。。")
except Exception:
yield from self.except_handle(user_name,
"交通银行---解析失败",
tell_msg="解析失败")
return
finally:
driver.quit()
def parse(self, response):
meta = response.meta
item = meta["item"]
username = item["username"]
driver = Ie(executable_path=self.settings['IE_EXECUTABLE_PATH'])
pid = driver.iedriver.process.pid
try:
# driver = self.getdriver(executable_path=self.settings["CHROME_EXECUTABLE_PATH"], browser_type="CHROME",
# use_proxy=False, change_proxy=False)
wait_60 = ui.WebDriverWait(driver, 60)
wait = ui.WebDriverWait(driver, 20)
driver.maximize_window()
driver.get(response.url)
for i in range(10):
try:
alert = driver.switch_to.alert
alert.accept()
sleep(1)
except Exception:
break
wait_60.until(EC.visibility_of_element_located((By.NAME, 'login-iframe')))
# 点击密码输入框
# driver.execute_script("inputPwd();")
# sleep(3)
# driver.execute_script("var obj_pwd=document.getElementById('password');"
# "obj_pwd.readOnly=false;"
# "obj_pwd.value='" + item["password"] + "';")
# driver.execute_script("var obj_pwd2=document.getElementById('Password');"
# "obj_pwd2.value='" + item["password"] + "';")
# 输入密码
# pwd_input = wait.until(EC.visibility_of_element_located((By.ID, 'OPassword')))
# pwd_input.send_keys(item["password"])
# 停顿2秒 准备模拟键盘输入
with Ddxoft(pid) as dd_opt:
# 输入用户名
driver.execute_script('login_iframe=document.getElementsByName("login-iframe")[0].contentDocument;'
'username_input=login_iframe.getElementById("LoginId");'
'username_input.focus();'
'username_input.value="' + username + '";')
# 切换到登录的框架
driver.switch_to.frame("login-iframe")
dd_opt.dd_tab()
for i in item['password']:
dd_opt.dd_keyboard(i)
sleep(0.5)
# 如果出现验证码
try:
# 输入验证码
validation_img = wait.until(EC.visibility_of_element_located((By.ID, 'tokenImg')))
left = validation_img.location['x']
top = validation_img.location['y']
right = validation_img.location['x'] + validation_img.size['width']
bottom = validation_img.location['y'] + validation_img.size['height']
photo_base64 = driver.get_screenshot_as_base64()
img_bytes = driver_screenshot_2_bytes(photo_base64, (left, top, right, bottom))
captcha_code = self.ask_image_captcha(img_bytes, username, file_type=".png")
token_input = wait.until(EC.visibility_of_element_located((By.ID, 'verCodeToken')))
token_input.send_keys(captcha_code)
except CaptchaTimeout:
raise
except Exception:
pass
# 点击登录
login_btn = wait.until(EC.visibility_of_element_located((By.ID, 'LoginButton')))
# login_btn.click()
driver.execute_script("Javascript: return doSubmit()")
# 获取出错信息
try:
error_div = wait.until(EC.visibility_of_element_located((By.ID, 'errInfo')))
msg = error_div.text
if msg != "":
yield from self.error_handle(username, msg, tell_msg=msg)
return
except TimeoutException:
pass
# 获取cookie
cookies = driver.get_cookies()
# 获取余额所在的页面
driver.get(self.balance_url)
balance_div = wait.until(EC.visibility_of_element_located((By.XPATH,
'//td[@id="CanUseBalanceShow_0"]/div')))
item["balance"] = balance_div.text.strip()
# 进入到明细查询页面
yield Request(
url=self.prequeryhistory_url,
callback=self.parse_prequeryhistory,
headers=self.headers,
cookies=cookies,
meta=meta,
dont_filter=True,
errback=self.err_callback
)
except CaptchaTimeout:
yield from self.error_handle(username, "浦发银行---等待验证码超时",
tell_msg="等待验证码超时，请刷新页面重试。。")
except Exception:
yield from self.except_handle(username, "浦发银行---登录数据解析异常")
finally:
driver.quit()
def parse(self, response):
item = response.meta["item"]
username = item["username"]
driver = self.load_page_by_webdriver(
"https://perbank.abchina.com/EbankSite/startup.do",
"//input[@id='username']")
pid = driver.iedriver.process.pid
try:
driver.maximize_window()
with Ddxoft(pid) as visual_keyboard:
driver.execute_script(
"username_input=document.getElementById('username');"
"username_input.value='%s';"
"username_input.focus();" % username)
visual_keyboard.dd_tab()
sleep(0.1)
for key in item["password"]:
visual_keyboard.dd_keyboard(key)
sleep(1.3) # 这样输入密码比较安全
# driver.execute_script("document.getElementById('userNameForm').pwdField.value='"
# + item["password"] + "';")
# 检查是否需要输入验证码
if driver.find_element_by_id("code").is_displayed():
with Ddxoft(pid) as visual_keyboard:
visual_keyboard.dd_tab()
captcha_image = driver.find_element_by_id("vCode")
location = captcha_image.location
size = captcha_image.size
left = location["x"] - 8
top = location["y"]
right = left + size["width"]
bottom = top + size["height"]
photo_base64 = driver.get_screenshot_as_base64()
captcha_body = driver_screenshot_2_bytes(
photo_base64, (left, top, right, bottom))
captcha_code = self.ask_image_captcha(captcha_body,
username,
file_type=".png")
with Ddxoft(pid) as visual_keyboard:
driver.execute_script(
"captcha_input=document.getElementById('code');"
"captcha_input.value='%s';"
"captcha_input.focus();" % captcha_code)
visual_keyboard.dd_tab()
try:
self.wait_xpath(driver,
"//span[@class='v-code-error right']",
raise_timeout=True,
timeout=3)
except TimeoutException:
yield from self.error_handle(username, "农业银行---图形验证码错误",
"图形验证码错误，请刷新页面重试。")
return
driver.execute_script("document.getElementById('logo').click();")
try:
self.wait_xpath(driver,
"//a[@id='logout_a']",
raise_timeout=True,
timeout=6)
except TimeoutException:
tell_msg = driver.find_element_by_id('powerpass_ie_dyn_Msg').text or \
driver.find_element_by_class_name("logon-error").get_attribute("title")
yield from self.error_handle(
username,
"农业银行---登录失败:(username:%s, password:%s) %s" %
(username, item["password"], '登录失败'),
tell_msg=tell_msg)
return
self.wait_xpath(driver, "//iframe[@id='contentFrame']")
driver.switch_to.frame("contentFrame")
self.wait_xpath(driver, "//div[@id='m-paycardcontent']")
item['balance'] = driver.find_element_by_id("dnormal").text
# 跳转到银行流水界面
onclick_fun_str = "toDetail('%s')" % username
self.wait_xpath(driver, '//a[@onclick="%s"]' % onclick_fun_str)
driver.switch_to.default_content()
driver.execute_script(
'document.getElementById("contentFrame").contentWindow.' +
onclick_fun_str + ';')
self.wait_xpath(driver, "//iframe[@id='contentFrame']")
driver.switch_to.frame("contentFrame")
self.wait_xpath(driver, '//input[@id="startDate"]')
driver.switch_to.default_content()
end_date = date.today()
# 有可能失败，执行两次
start_date_str = (end_date - self.date_delta).strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
for i in range(2):
driver.execute_script(
'data_iframe=document.getElementById("contentFrame").contentDocument;'
'data_iframe.getElementById("startDate").value="%s";'
'data_iframe.getElementById("endDate").value="%s";' %
(start_date_str, end_date_str))
sleep(0.6)
driver.execute_script(
'data_iframe.getElementById("btn_query").click();')
item["trade_records"] = self.parse_item_from_webpage(driver)
yield from self.crawling_done(item)
except CaptchaTimeout:
yield from self.error_handle(username,
"农业银行---等待验证码超时",
tell_msg="等待验证码超时，请刷新页面重试。。")
except Exception:
yield from self.except_handle(username,
msg="农业银行---错误信息",
tell_msg="银行流水数据爬取失败，请刷新页面重试!")
finally:
driver.quit()
def parse(self, response):
meta = response.meta
item = meta["item"]
username = item["username"]
driver = Ie(executable_path=self.settings['IE_EXECUTABLE_PATH'])
pid = driver.iedriver.process.pid
try:
driver.maximize_window()
wait = ui.WebDriverWait(driver, 20)
driver.get(response.url)
username = wait.until(EC.visibility_of_element_located((By.ID, 'textfield')))
username.send_keys(username)
# 停顿2秒 准备模拟键盘输入
with Ddxoft(pid) as visual_keyboard:
visual_keyboard.dd_tab()
visual_keyboard.dd_tab()
for i in item['password']:
visual_keyboard.dd_keyboard(i)
sleep(0.5)
# 输入验证码
validation_img = wait.until(EC.visibility_of_element_located((By.ID, '_tokenImg')))
left = validation_img.location['x']
top = validation_img.location['y']
right = validation_img.location['x'] + validation_img.size['width']
bottom = validation_img.location['y'] + validation_img.size['height']
photo_base64 = driver.get_screenshot_as_base64()
img_bytes = driver_screenshot_2_bytes(photo_base64, (left, top, right, bottom))
captcha_code = self.ask_image_captcha(img_bytes, username, file_type=".png")
token_input = wait.until(EC.visibility_of_element_located((By.ID, '_vTokenName')))
token_input.send_keys(captcha_code)
button = driver.find_element_by_id('button')
button.click()
# 判断是否登录成功
try:
error_info = wait.until(EC.visibility_of_element_located((By.ID, 'EEE')))
msg = error_info.text
if msg != "":
yield from self.error_handle(username, msg, tell_msg=msg)
return
except TimeoutException:
pass
cookies = driver.get_cookies()
yield Request(
url=self.account_url,
callback=self.parse_act_list,
headers=self.headers,
cookies=cookies,
meta=meta,
dont_filter=True,
errback=self.err_callback
)
except CaptchaTimeout:
yield from self.error_handle(username, "邮政银行---等待验证码超时",
tell_msg="等待验证码超时，请刷新页面重试。。")
except Exception:
yield from self.except_handle(username, "邮政银行---登录数据解析异常")
finally:
driver.quit()
def parse(self, response):
meta = response.meta
item = meta['item']
mail_type = meta['mail_type']
username = item["username"]
password = item["password"]
url = self._start_url_
driver = self.load_page_by_webdriver(url, '//iframe')
try:
if mail_type == '163.com':
host = 'http://mail.163.com/'
iframe = driver.find_element_by_tag_name('iframe')
elif mail_type == '126.com':
host = 'http://mail.126.com/'
tab_126 = driver.find_element_by_class_name('item-126')
tab_126.click()
driver.implicitly_wait(1)
iframe = driver.find_elements_by_tag_name('iframe')[1]
elif mail_type == 'yeah.net':
host = 'http://mail.yeah.net/'
tab_yeah = driver.find_element_by_class_name('item-yeah')
tab_yeah.click()
driver.implicitly_wait(1)
iframe = driver.find_elements_by_tag_name('iframe')[1]
else:
err_message = '不支持非163、126、yeah邮箱系统'
yield from self.error_handle(
username,
msg="163---不支持此邮箱：(username:%s, password:%s) %s" %
(username, password, err_message),
tell_msg=err_message)
return
driver.execute_script(
'''document.querySelector("a[data-style='7']").click();''')
driver.switch_to.frame(iframe)
self.wait_xpath(driver, "//a[@id='dologin']")
driver.execute_script(
'document.getElementsByName("email")[0].value="%s";'
'document.getElementsByName("password")[0].value="%s";'
'document.getElementById("dologin").click();' %
(username.split('@', 1)[0], password))
driver.implicitly_wait(1)
# 切换到表单
try:
if driver.find_element_by_id('ScapTcha'):
sleep(1)
validat_img = False
for i in range(self.captcha_retry_time):
img_desc = driver.find_element_by_xpath(
'//div[@class="ncpt_panel"]'
'/div[@class="ncpt_hint_txt"]').text
sleep(0.5)
img_url = driver.find_element_by_xpath(
'//div[@class="ncpt_pzzClick"]'
'/img').get_attribute('src')
x1, y1, x2, y2, x3, y3 = self.send_pic_get_ponint6_to_ssdb(
img_url, img_desc, username)
driver.find_element_by_id('ScapTcha').click()
e163 = E_3click(driver)
e163.click_3_char('//div[@class="ncpt_pzzClick"]/img',
x1, y1, x2, y2, x3, y3)
driver.execute_script(
"document.getElementById('dologin').click();")
sleep(1)
if "帐号或密码" in driver.find_element_by_class_name(
"ferrorhead").text:
err_message = '账号或密码错误'
yield from self.error_handle(
username,
msg="163---登录失败：(username:%s, password:%s) %s"
% (username, password, err_message),
tell_msg=err_message)
return
if "成功" in driver.find_element_by_xpath(
'//div[@class="ncpt_panel"]'
'/div[@class="ncpt_hint_txt"]').text:
validat_img = True
break
elif "点击" in driver.find_element_by_xpath('//div[@class="ncpt_panel"]'
'/div[@class="ncpt_hint_txt"]').text \
or "失败" in driver.find_element_by_xpath('//div[@class="ncpt_panel"]'
'/div[@class="ncpt_hint_txt"]').text:
self.wait_xpath(
driver,
'//div[@class="ncpt_panel"]/div[contains(text(),"点击")]'
)
else:
validat_img = True
break
if not validat_img:
err_message = "点击验证错误四次，请重试！"
yield from self.error_handle(
username,
msg="163---登录失败：(username:%s, password:%s) %s" %
(username, password, err_message),
tell_msg=err_message)
return
except NoSuchElementException:
pass
try:
if 'errorAlert-show' in driver.find_element_by_id(
'errorAlert').get_attribute('class'):
err_message = '账号或密码错误'
yield from self.error_handle(
username,
msg="163---登录失败：(username:%s, password:%s) %s" %
(username, password, err_message),
tell_msg=err_message)
return
except NoSuchElementException:
pass
try:
if driver.find_element_by_name("phonecode"):
click_btn = driver.find_element_by_class_name('getsmscode')
click_btn.click()
sms_code = self.ask_sms_captcha(username)
driver.find_element_by_name("phonecode").send_keys(
sms_code)
driver.find_element_by_class_name("u-loginbtn").click()
sleep(1)
except NoSuchElementException:
pass
curr_url = driver.current_url
# self.logger.debug('++++\ncurrent_url :{0}\n+++++'.format(curr_url))
# 跳转多种页面。
if url == curr_url or 'errorType' in curr_url:
# 说明未登录成功
err_message = '账号或密码错误'
yield from self.error_handle(
username,
msg="163---登录失败：(username:%s, password:%s) %s" %
(username, password, err_message),
tell_msg=err_message)
return
if 'clearlimit' in curr_url:
# 说明未登录成功跳转到检查网站
phone = self.ask_extra_captcha(username)
# self.logger.debug('输入电话:{0}'.format(phone))
driver.execute_script(
"document.getElementById('password').value='{0}';"
"document.getElementById('mobile').value='{1}';".format(
password, phone))
driver.find_element_by_class_name('yzm_btn').click()
driver.implicitly_wait(0.5)
try:
if driver.find_element_by_id(
'dg-usercheckcode').is_displayed():
# cookies = get_cookies_dict_from_webdriver(driver)
# self.logger.debug('cookies:{0}'.format(cookies))
# headers = {
# 'Accept': '*/*',
# 'Accept-Encoding': 'gzip, deflate',
# 'Accept-Language': 'zh-CN,zh;q=0.9',
# 'Connection': 'keep-alive',
# 'Host': 'reg.163.com',
# 'Referer': 'http://reg.163.com/clearlimit/check.jsp?username={0}'
# '&url=http%3A%2F%2Fentry.mail.163.com%2Fcoremail%2Ffcg%2Fntesdoor2'
# '%3Fdf%3Dwebmail163%26from%3Dweb%26style%3D-1%26product%3Dmail163'
# '%26uid%3D{1}'.format(username, url_parse.quote(username)),
# 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36',
# 'X-Requested-With': 'XMLHttpRequest',
# }
# get_id = get_content_by_requests('http://reg.163.com/services/getid',headers=headers,cookie_jar=cookies).decode()
# self.logger.debug('get_id:{0}'.format(get_id))
# capcha_body = get_content_by_requests('http://reg.163.com/services/getimg?v={0}&num=6&type=2&id={1}'.format(get_js_time(),get_id),cookie_jar=cookies,headers=headers)
# self.logger.debug('capcha_body:{0}'.format(capcha_body))
wait = ui.WebDriverWait(driver, 20)
validation_img = wait.until(
EC.visibility_of_element_located(
(By.ID, 'dg-checkCode')))
left = validation_img.location['x']
top = validation_img.location['y']
right = validation_img.location[
'x'] + validation_img.size['width']
bottom = validation_img.location[
'y'] + validation_img.size['height']
photo_base64 = driver.get_screenshot_as_base64()
capcha_body = driver_screenshot_2_bytes(
photo_base64, (left, top, right, bottom))
captcha_code = self.ask_image_captcha(
capcha_body, username)
driver.find_element_by_id(
'dg-usercheckcode').send_keys(captcha_code)
driver.find_element_by_class_name('iDialogBtn').click()
driver.implicitly_wait(0.5)
try:
# self.logger.debug('err?:{0}'.format(driver.find_element_by_id('dg-err')
# .get_attribute('class')))
if 'err' in driver.find_element_by_id(
'dg-err').get_attribute('class'):
yield from self.error_handle(
username,
msg="验证码输入错误",
tell_msg="验证码输入错误")
return
# self.logger.debug('p {0}'.format(driver.find_element_by_tag_name('p')))
if driver.find_element_by_tag_name('p'):
yield from self.error_handle(
username,
msg=driver.find_element_by_tag_name(
'p').text,
tell_msg=driver.find_element_by_tag_name(
'p').text)
return
except Exception:
alert = driver.switch_to.alert
self.logger.debug("Alert:%s" % alert.text)
if '请查收' not in alert.text:
yield from self.error_handle(
username,
msg=alert.text,
tell_msg=alert.text)
return
alert.accept()
driver.switch_to.default_content()
except Exception:
pass
# driver.get_screenshot_as_file("/logs/phone.jpg")
sms_code = self.ask_sms_captcha(username)
# self.logger.debug('输入验证码:{0}'.format(sms_code))
# self.logger.debug('page_source:{0}'.format(driver.page_source))
driver.find_element_by_id('mobcheckcode').send_keys(sms_code)
# driver.execute_script("document.getElementById('mobcheckcode').value='{0}';".format(sms_code))
driver.find_element_by_id('loginBtn').click()
driver.implicitly_wait(2)
if len(driver.find_element_by_id('eHint').text) > 6:
yield from self.error_handle(
username,
msg=driver.find_element_by_id('eHint').text,
tell_msg=driver.find_element_by_id('eHint').text)
return
# driver.get_screenshot_as_file("/logs/sms.jpg")
driver.implicitly_wait(1)
folder_str = self.folders_pattern.search(driver.page_source)
if not folder_str:
yield from self.error_handle(username,
msg="没有文件夹",
tell_msg="出现未知异常")
return
else:
folder_str = folder_str.group(1)
folder_list = self.folder_id_pattern.findall(folder_str)
folder_list.remove('2') # 草稿箱
folder_list.remove('3') # 已发送
folder_list.remove('6') # 病毒邮件
folder_str = ''.join('<int>' + str(x) + '</int>'
for x in folder_list)
# self.logger.debug('driver_current_url: %s' % driver.current_url)
# sid = driver.current_url.split('?', 1)[-1].split('&', 1)[0]
sid = parse_qs(urlsplit(driver.current_url).query)["sid"][0]
headers = {
'Connection':
'keep-alive',
'Accept-Encoding':
'gzip, deflate',
'Accept-Language':
'zh-CN,zh;q=0.8',
'Content-type':
'application/x-www-form-urlencoded',
'Host':
'mail.%s' % mail_type,
'Origin':
'http://mail.%s' % mail_type,
'Referer':
'http://mail.%s/js6/main.jsp?sid=%s&df=mail163_letter' %
(mail_type, sid),
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3050.3 Safari/537.36',
}
meta['sid'] = sid
meta['host'] = host
cookie = driver.get_cookies()
# meta['folder_str'] = folder_str
# mbox_url = '{0}js6/main.jsp?sid={1}&df=mail{2}_letter#module=mbox.ListModule' \
# '%7C%7B%22fid%22%3A1%2C%22order%22%3A%22date%22%2C%22' \
# 'desc%22%3Atrue%7D'.format(host, sid, mail_type[mail_type.index(".")])
# yield Request(url=mbox_url,
# headers=headers,
# cookies=cookie,
# meta=meta,
# callback=self.parse_mbox,
# dont_filter=True,
# errback=self.err_callback)
search_url = host + 'js6/s?sid=' + sid + '&func=mbox:searchMessages'
data = {
'var':
'<?xml version="1.0"?><object><array name="conditions"><object>'
'<array name="conditions"><object><string name="field">subject</string>'
'<string name="operator">contains</string><string name="operand">{0}</string>'
'<boolean name="ignoreCase">true</boolean></object></array><string name="operator">or'
'</string></object></array><int name="windowSize">{1}</int><string name="order">date'
'</string><boolean name="desc">true</boolean><boolean name="returnTag">true</boolean>'
'<boolean name="recursive">true</boolean><array name="fids">{2}</array>'
'</object>'.format(CREDIT_CARD_KEYWORD, self.PAGE_PER_COUNT,
folder_str),
}
yield FormRequest(url=search_url,
callback=self.parse_search,
formdata=data,
meta=meta,
headers=headers,
cookies=cookie,
dont_filter=True,
errback=self.err_callback)
except Exception:
yield from self.except_handle(username,
msg="邮箱登录异常",
tell_msg="邮箱登录异常")
finally:
driver.quit()