def _get_captcha_code_by_chrome(self, driver, username): captcha_xpath = "//img[@id='rndCodeCue']" self.wait_xpath(driver, captcha_xpath) captcha_element = driver.find_element_by_xpath(captcha_xpath) location = captcha_element.location size = captcha_element.size left = location["x"] top = location["y"] photo_base64 = driver.get_screenshot_as_base64() captcha_body = driver_screenshot_2_bytes( photo_base64, (left, top, left + size["width"], top + size["height"])) return self.ask_image_captcha(captcha_body, username, file_type=".png")
def __get_captcha_code(self, driver, username, flag=True): if flag: driver.switch_to.frame('ICBC_login_frame') else: driver.switch_to.frame('integratemainFrame') driver.switch_to.frame('VerifyimageFrame') captcha_xpath = "//img[@title='点击图片可刷新']" self.wait_xpath(driver, captcha_xpath) location = driver.find_element_by_xpath( captcha_xpath).location_once_scrolled_into_view left = location["x"] + 8 top = location["y"] driver.switch_to.default_content() photo_base64 = driver.get_screenshot_as_base64() captcha_body = driver_screenshot_2_bytes( photo_base64, (left, top, left + 90, top + 36)) return self.ask_image_captcha(captcha_body, username, file_type=".png")
def login(self, driver, wait, response): meta = response.meta try: main_div_xpath = "//form[@id='login_form2']" driver.get(response.url) # 识别验证码 wait.until(lambda dr: dr.find_element_by_xpath( main_div_xpath + "//div[@class='qrcode']/img").is_displayed()) captcha_ele = driver.find_element_by_xpath( main_div_xpath + "//div[@class='qrcode']/img") location = captcha_ele.location size = captcha_ele.size # 获取验证码的长宽 left, top = location['x'], location['y'] photo_base64 = driver.get_screenshot_as_base64() temp = driver_screenshot_2_bytes( photo_base64, (left, top, left + size['width'], top + size['height'])) captcha_code = recognize_captcha_auto(temp, digits_only=True, del_noise=True) if len(captcha_code) != 3: raise CaptcherInvalidException() _cookies = get_cookies_dict_from_webdriver(driver) # 填写input input_capimg = driver.find_element_by_xpath( main_div_xpath + "//input[@type='hidden'][1]") input_cookiename = driver.find_element_by_xpath( main_div_xpath + "//input[@type='hidden'][2]") input_username = driver.find_element_by_xpath( main_div_xpath + "//input[@type='text'][1]") input_password = driver.find_element_by_xpath( main_div_xpath + "//input[@type='password'][1]") input_qrcode = driver.find_element_by_xpath( main_div_xpath + "//div[@class='qrcode']/input") item = meta["item"] formdata = { input_capimg.get_attribute("name"): input_capimg.get_attribute("value"), input_cookiename.get_attribute("name"): input_cookiename.get_attribute("value"), input_username.get_attribute("name"): item["username"], input_password.get_attribute("name"): item["password"], input_qrcode.get_attribute("name"): captcha_code } return FormRequest( "http://jypt.cdhrss.gov.cn:8048/mlogin_action.php?id=1", meta=meta, cookies=_cookies, formdata=formdata, dont_filter=True, callback=self.login_step2, errback=self.err_callback) except CaptcherInvalidException: raise except Exception: raise LoginException(response.url)
def parse(self, response): meta = response.meta item = meta['item'] username = item["username"] driver = self.load_page_by_webdriver(self._start_url_) sleep(0.5) try: try: driver.execute_script("""$('input[ng-model="account"]').val('%s'); $('input[ng-model="account"]').change();""" % username) sleep(0.2) driver.execute_script("""$('input[ng-model="pwd"]').val('%s'); $('input[ng-model="pwd"]').change();""" % item["password"]) sleep(0.2) driver.execute_script("document.getElementsByClassName('btn-login fontFamily')[0].click();") sleep(2) if driver.current_url == self._start_url_: captcha_image = driver.find_element_by_xpath('//div[@ng-show="needCaptcha"]/img') location = captcha_image.location size = captcha_image.size left = location["x"] - 8 top = location["y"] right = left + size["width"] bottom = top + size["height"] photo_base64 = driver.get_screenshot_as_base64() captcha_body = driver_screenshot_2_bytes(photo_base64, (left, top, right, bottom)) captcha_code = self.ask_image_captcha(captcha_body, username, file_type=".png") driver.execute_script("""$('input[ng-model="captcha"]').val('%s'); $('input[ng-model="captcha"]').change();""" % captcha_code) driver.execute_script("document.getElementsByClassName('btn-login fontFamily')[0].click()") # 登录成功之后 直接访问json页面, 提取对应数据进行筛选 # sleep(0.5) except TypeError: pass except AttributeError: # 跳转页面会出现异常,这个异常可忽略. pass driver.get(self.search_url % (0, get_js_time(), self.keyword)) result_str = self.mail_list_pattern.search(driver.page_source).group(1) result = json_loads(result_str) if result['msg'] != '登录超时,请重新登录': bill_records = item["bill_records"] the_data = result['data'] bill_records.extend(self._get_bill_records(driver, the_data['list'])) page_step = self.page_step page = (the_data['total'] + page_step - 1) // page_step for pa in range(1, page): # 过滤掉第一页 driver.get(self.search_url % (pa * page_step, get_js_time(), self.keyword)) result_str = self.mail_list_pattern.search(driver.page_source).group(1) bill_records.extend(self._get_bill_records(driver, json_loads(result_str)['data']['list'])) yield from self.crawling_done(item) else: yield from self.error_handle(username, "搜狐邮箱---等待验证码超时", tell_msg="等待验证码超时,请刷新页面重试。。") except Exception: yield from self.except_handle(username, "搜狐邮箱---抓取异常", tell_msg="请刷新页面重试。。") finally: driver.quit()
def parse(self, response): meta = response.meta item = meta["item"] username = item["username"] driver = self.getdriver( executable_path=self.settings["CHROME_EXECUTABLE_PATH"], browser_type="CHROME_HEADLESS") try: wait = ui.WebDriverWait(driver, 10) driver.get(response.url) # 输入用户名 username_input = wait.until( EC.visibility_of_element_located( (By.ID, 'txt_username_79443'))) username_input.send_keys(username) username_input.send_keys(Keys.TAB) # 输入密码 try: pwd_input = wait.until( EC.visibility_of_element_located( (By.ID, 'input_div_password_79445_1'))) except TimeoutException: pwd_input = wait.until( EC.visibility_of_element_located( (By.ID, 'input_div_password_79445'))) pwd_input.send_keys(item["password"]) self.click_hidemsgbox(wait) # 输入验证码 try: try: validation_img = wait.until( EC.visibility_of_element_located( (By.ID, 'captcha_debitCard'))) except TimeoutException: validation_img = wait.until( EC.visibility_of_element_located((By.ID, 'captcha'))) left = validation_img.location['x'] top = validation_img.location['y'] right = validation_img.location['x'] + validation_img.size[ 'width'] bottom = validation_img.location['y'] + validation_img.size[ 'height'] photo_base64 = driver.get_screenshot_as_base64() img_bytes = driver_screenshot_2_bytes( photo_base64, (left, top, right, bottom)) captcha_code = self.ask_image_captcha(img_bytes, username, file_type=".png") self.click_hidemsgbox(wait) try: validation_input = wait.until( EC.visibility_of_element_located( (By.ID, 'txt_captcha_741012'))) except TimeoutException: validation_input = wait.until( EC.visibility_of_element_located( (By.ID, 'txt_captcha_79449'))) validation_input.send_keys(captcha_code) self.click_hidemsgbox(wait) except Exception: pass # 点击登录 try: submit_input = wait.until( EC.visibility_of_element_located((By.ID, 'btn_49_741014'))) except TimeoutException: submit_input = wait.until( EC.visibility_of_element_located( (By.ID, 'btn_login_79676'))) submit_input.click() # 判断是否登录成功 try: error_info = wait.until( EC.visibility_of_element_located((By.ID, 'msgContent'))) msg = error_info.text if msg != "": yield from self.error_handle(username, msg, tell_msg=msg) return except TimeoutException: pass if username.isdigit() and len(username) == 19: # 使用webdriver的方式 # 查询余额 balance_td = wait.until( EC.visibility_of_element_located( (By.XPATH, '//*[@id="cardMain"]/div/div[2]' '/table/tbody/tr/td[4]'))) item["balance"] = balance_td.text.strip() # 选择查询交易明细 trade_detail = wait.until( EC.visibility_of_element_located( (By.ID, 'div_transaction_details_740993'))) trade_detail.click() # 选择日期 start_date = wait.until( EC.visibility_of_element_located( (By.ID, 'date_start_date_740972'))) begin_date = (date.today() - timedelta(days=180)).strftime('%Y/%m/%d') start_date.clear() start_date.send_keys(begin_date) # 点击空白 null_click = wait.until( EC.visibility_of_element_located( (By.ID, 'sel_pleaseselectaccount_740896'))) null_click.click() # 点击查询按钮 query_input = wait.until( EC.visibility_of_element_located((By.ID, 'btn_49_740974'))) query_input.click() trade_records = item["trade_records"] # 获取总页码 page_count_em = driver.find_element_by_xpath( '//*[@id="pager"]/ul/li[2]/em[2]') page_count = int(page_count_em.text) next_count = 0 while next_count < page_count: # 滚到顶部 driver.execute_script("window.scrollTo(0,0)") wait.until( EC.visibility_of_element_located( (By.XPATH, '//*[@id="debitCardTransDetail_table"]' '/table/tbody/tr'))) tr_list = driver.find_elements_by_xpath( '//*[@id="debitCardTransDetail_table"]/table/tbody/tr') for tr in tr_list: td = tr.find_elements_by_xpath("td") trade_income = td[6].text trade_outcome = td[7].text tmp_dict = { "trade_date": td[0].text, "trade_remark": td[1].text, "trade_acceptor_account": td[2].text, "trade_acceptor_name": td[3].text, "trade_currency": td[4].text, "trade_income": trade_income, "trade_outcome": trade_outcome, "trade_amount": trade_income or ("-" + trade_outcome), "trade_balance": td[8].text, "trade_channel": td[9].text, # 交易渠道 } trade_records.append(tmp_dict) next_count += 1 # 滚到页面底部才有下一页按钮 driver.execute_script( "window.scrollTo(0,document.body.scrollHeight)") next_pager = wait.until( EC.visibility_of_element_located( (By.CLASS_NAME, 'next'))) next_pager.click() sleep(1) driver.execute_script("window.scrollTo(0,0)") # 退出网银 pager_quit = wait.until( EC.visibility_of_element_located((By.ID, 'a_59_1168'))) pager_quit.click() # 抓取完成 yield from self.crawling_done(item) else: cookies = driver.get_cookies() # 点击银行账户链接 my_account = wait.until( EC.visibility_of_element_located( (By.XPATH, '//a[@key="MyAccount"]'))) my_account.click() my_balance = wait.until( EC.visibility_of_element_located( (By.XPATH, '//a[@title="余额"]'))) # 获取accountid accountid = my_balance.get_attribute('accountid') my_balance.click() # 获取余额 item["balance"] = wait.until( EC.visibility_of_element_located( (By.XPATH, '//td[@lan="num"]'))).text begin_date = (date.today() - timedelta(days=180)).strftime('%Y/%m/%d') end_date = date.today().strftime('%Y/%m/%d') # 下载账单 download_url = "%s?accountId=%s¤cy=%s&cashRemit=&startDate=%s&endDate=%s" \ % (self.download_url, accountid, "001", begin_date, end_date) yield Request(url=download_url, callback=self.parse_download, headers=self.headers, meta=meta, cookies=cookies, dont_filter=True, errback=self.err_callback) except CaptchaTimeout: yield from self.error_handle(username, "中国银行---等待验证码超时", tell_msg="等待验证码超时,请刷新页面重试。。") except Exception: yield from self.except_handle(username, "中国银行---数据解析异常") finally: driver.quit()
def parse(self, response): item = response.meta["item"] username = item["username"] driver = self.getdriver( executable_path=self.settings["IE_EXECUTABLE_233_PATH"], browser_type="IE") try: wait = ui.WebDriverWait(driver, 20) driver.get(response.url) wait.until(lambda dr: dr.find_element_by_id("loginId")) driver.maximize_window() pid = driver.iedriver.process.pid with Ddxoft(pid) as visual_keyboard: driver.execute_script( 'user_input=document.getElementById("loginId");' 'user_input.focus();' 'user_input.value="' + username + '";') visual_keyboard.dd_tab() sleep(0.1) for key in item["password"]: visual_keyboard.dd_keyboard(key) sleep(0.1) # 检查是否需要输入验证码 captcha_input = driver.find_element_by_id("captcha") if captcha_input and captcha_input.is_displayed(): captcha_image = driver.find_element_by_id("verifyImg") location = captcha_image.location size = captcha_image.size left = location["x"] - 5 top = location["y"] right = left + size["width"] bottom = top + size["height"] photo_base64 = driver.get_screenshot_as_base64() captcha_body = driver_screenshot_2_bytes( photo_base64, (left, top, right, bottom)) captcha_code = self.ask_image_captcha(captcha_body, username, file_type=".png") captcha_input.send_keys(captcha_code) driver.execute_script( 'document.getElementById("captcha").value="' + captcha_code + '";') driver.execute_script( 'document.getElementById("loginButton").click();') wait.until(lambda dr: dr.find_element_by_id("_b01").is_displayed()) driver.execute_script( 'document.getElementById("_b01").children[1].style.display="block";' 'document.getElementById("_b01").children[1].style.left="-49px";' ) sleep(5) # with open("webpage.html", "w", encoding="utf-8") as file # file.write(driver.page_source) # 跳转到银行流水界面 trade_detail_btn = driver.find_element_by_xpath( "//a[contains(text(),'交易明细')]") if trade_detail_btn: trade_detail_btn.click() wait.until(lambda dr: dr.find_element_by_id("windowContainer"). is_displayed()) driver.switch_to.frame( driver.find_element_by_id( "windowContainer").find_element_by_xpath("//iframe")) start_date_input = driver.find_element_by_id("beginDate") end_date_input = driver.find_element_by_id("endDate") submit_input = driver.find_element_by_id("advancedQueryTable")\ .find_element_by_xpath("//a[contains(text(),'查询')]") end_date = date.today() trade_detail = [] for i in range(4): start_date = self.get_theday_3_month_ago(end_date) try: start_date_input.clear() start_date_input.send_keys( start_date.strftime("%Y-%m-%d")) end_date_input.clear() end_date_input.send_keys(end_date.strftime("%Y-%m-%d")) submit_input.click() wait.until(lambda dr: dr.find_element_by_id( "advancedQueryTable").is_displayed()) trade_detail.extend( self.parse_item_from_webpage(driver, wait, item)) except Exception: pass end_date = start_date item["trade_records"] = trade_detail yield from self.crawling_done(item) else: yield from self.error_handle(username, msg="广发银行---错误信息:在页面中找不到[交易明细]按钮", tell_msg="银行流水数据爬取失败,请刷新页面重试!") except CaptchaTimeout: yield from self.error_handle(username, "广发银行---等待验证码超时", tell_msg="等待验证码超时,请刷新页面重试。。") except Exception: yield from self.except_handle(username, msg="广发银行---错误信息:", tell_msg="银行流水数据爬取失败,请刷新页面重试!") finally: driver.quit()
def parse(self, response): meta = response.meta item = meta['item'] username = item['username'] password = item['password'] driver = self.load_page_by_webdriver(response.url) try: driver.implicitly_wait(1) driver.execute_script( 'document.getElementsByClassName("menu_item")[0].click();') driver.implicitly_wait(1) driver.execute_script( 'document.getElementsByName("logonId")[0].value="%s";' 'document.getElementsByName("getSms")[0].click();' % username) page = str(bs_format(driver.page_source, "lxml")) if "短信已发送" in page: self.logger.info("--->短信已发送") sms_code = self.ask_sms_captcha(username) self.logger.info("短信验证码: %s" % sms_code) # 输入验证码 validation_img = driver.find_element_by_xpath( './/*[@alt="checkCode"]') left = validation_img.location['x'] top = validation_img.location['y'] right = validation_img.location['x'] + validation_img.size[ 'width'] bottom = validation_img.location['y'] + validation_img.size[ 'height'] photo_base64 = driver.get_screenshot_as_base64() img_bytes = driver_screenshot_2_bytes( photo_base64, (left, top, right, bottom)) captcha_code = self.ask_image_captcha(img_bytes, username, file_type=".png") self.logger.info("图片验证码: %s" % captcha_code) driver.execute_script( 'document.getElementsByName("smsPassword")[0].value="%s";' 'document.getElementsByName("password")[0].value="%s";' 'document.getElementsByName("checkCode")[0].value="%s";' 'document.getElementsByName("login")[0].click();' % (sms_code, password, captcha_code)) driver.implicitly_wait(1) res_page = str(bs_format(driver.page_source, "lxml")) if "登录信息输入有误" in res_page: yield from self.error_handle( username, msg="广发银行---登录信息输入有误", tell_msg="银行流水数据爬取失败,请刷新页面重试!") elif "accmanage" in res_page: self.logger.info("登录成功") cookies = driver.get_cookies() tran_his_url = "https://wap.cgbchina.com.cn/tranHistoryQueryInput.do?currentMenuId=101002" yield Request(tran_his_url, callback=self.parse_balance_info, meta=meta, headers=self.headers, cookies=cookies, dont_filter=True, errback=self.err_callback) else: yield from self.error_handle( username, msg="广发银行---未知错误", tell_msg="银行流水数据爬取失败,请刷新页面重试!") else: yield from self.error_handle(username, msg="广发银行---短信发送失败", tell_msg="银行流水数据爬取失败,请刷新页面重试!") except CaptchaTimeout: yield from self.error_handle(username, "广发银行---等待验证码超时", tell_msg="等待验证码超时,请刷新页面重试。。") except Exception: yield from self.except_handle(username, msg="广发银行---登录失败", tell_msg="银行流水数据爬取失败,请刷新页面重试!") finally: driver.quit()
def parse(self, response): meta = response.meta item = meta["item"] user_name = item['username'] driver = self.load_page_by_webdriver(response.url) try: driver.maximize_window() # username_input = driver.find_element_by_id('alias') # username_input.send_keys(user_name) pid = driver.iedriver.process.pid with Ddxoft(pid) as dd_opt: driver.execute_script( 'document.getElementById("alias").value="{0}";' 'document.getElementById("alias").focus()'.format( user_name)) sleep(0.5) dd_opt.dd_tab() sleep(0.5) for i in item['password']: dd_opt.dd_keyboard(i) try: captcha_img = driver.find_element_by_class_name( 'captchas-img-bg') input_captcha = driver.find_element_by_id('input_captcha') if captcha_img.is_displayed(): photo_base64 = driver.get_screenshot_as_base64() left = captcha_img.location['x'] - 8 top = captcha_img.location['y'] right = captcha_img.location['x'] + captcha_img.size[ 'width'] - 8 bottom = captcha_img.location['y'] + captcha_img.size[ 'height'] captcha_body = driver_screenshot_2_bytes( photo_base64, (left, top, right, bottom)) captcha_code = self.ask_image_captcha(captcha_body, user_name, file_type=".png") # driver.execute_script('document.getElementById("input_captcha").value="{0}";' # .format(captcha_code)) input_captcha.send_keys(captcha_code) except NoSuchElementException: pass butt_submit = driver.find_element_by_id('login') self.element_click_three_times(butt_submit) sleep(3) try: driver.find_element_by_id('login') yield from self.error_handle( user_name, "交通银行---登录失败:(username:%s, password:%s) %s" % (user_name, item["password"], '账号、密码错误或模拟键盘操作失败,返回登录页面'), tell_msg="账号、密码错误或模拟键盘操作失败") return except NoSuchElementException: pass try: # 有可能没有验证手机 sleep(1) btnSendCode = driver.find_element_by_id('authSMSSendBtn') self.element_click_three_times(btnSendCode) sms_code = self.ask_sms_captcha(user_name) sms_code_input = driver.find_element_by_id('mobileCode') sms_code_input.send_keys(sms_code) sleep(0.5) submit = driver.find_element_by_id('btnConf2') self.element_click_three_times(submit) sleep(1) except NoSuchElementException: pass # 提交设置为常用电脑 try: self.element_click_three_times( driver.find_element_by_xpath( '//input[@checked="checked"]')) self.element_click_three_times( driver.find_element_by_id('next')) sleep(0.5) except NoSuchElementException: pass self.wait_xpath(driver, '//iframe[@id="frameMain"]') driver.switch_to.frame('frameMain') driver.execute_script( 'document.getElementById("search").value="账户明细查询";') # driver.find_element_by_id('search').send_keys('账户明细查询') self.element_click_three_times( driver.find_element_by_class_name('search_btn')) self.wait_xpath(driver, '//iframe[@id="tranArea"]') frame = driver.find_element_by_id('tranArea') # 转到 主工作区 iframe 里面 driver.switch_to.frame(frame) # 搜索框中的交易明细查询 menucode为P002000 if driver.find_elements_by_xpath('//td[@menucode="P002000"]/a'): self.element_click_three_times( driver.find_element_by_xpath( '//td[@menucode="P002000"]/a')) sleep(1) fromday = (date.today() - self.date_delta).strftime("%Y%m%d") driver.execute_script( '$("#startDate").datepicker("setDate","{0}");'.format( fromday)) submit_date = driver.find_element_by_id('btnQry2') self.element_click_three_times(submit_date) sleep(0.5) self.wait_xpath(driver, '//tbody[@id="recordtbody"]') page_source = driver.page_source result = self.__get_page_detail(page_source) for i in range(100): try: pageDown_btn = driver.find_element_by_id('pageDown') except NoSuchElementException: break if pageDown_btn.is_displayed(): self.element_click_three_times(pageDown_btn) sleep(1) page_source = driver.page_source result.extend(self.__get_page_detail(page_source)) else: break item['trade_records'] = result yield from self.crawling_done(item) else: yield from self.error_handle(user_name, "交通银行---未搜索到交易明细", tell_msg="信息获取异常") except CaptchaTimeout: yield from self.error_handle(user_name, "交通银行---等待验证码超时", tell_msg="等待验证码超时,请刷新页面重试。。") except Exception: yield from self.except_handle(user_name, "交通银行---解析失败", tell_msg="解析失败") return finally: driver.quit()
def parse(self, response): meta = response.meta item = meta["item"] username = item["username"] driver = Ie(executable_path=self.settings['IE_EXECUTABLE_PATH']) pid = driver.iedriver.process.pid try: # driver = self.getdriver(executable_path=self.settings["CHROME_EXECUTABLE_PATH"], browser_type="CHROME", # use_proxy=False, change_proxy=False) wait_60 = ui.WebDriverWait(driver, 60) wait = ui.WebDriverWait(driver, 20) driver.maximize_window() driver.get(response.url) for i in range(10): try: alert = driver.switch_to.alert alert.accept() sleep(1) except Exception: break wait_60.until(EC.visibility_of_element_located((By.NAME, 'login-iframe'))) # 点击密码输入框 # driver.execute_script("inputPwd();") # sleep(3) # driver.execute_script("var obj_pwd=document.getElementById('password');" # "obj_pwd.readOnly=false;" # "obj_pwd.value='" + item["password"] + "';") # driver.execute_script("var obj_pwd2=document.getElementById('Password');" # "obj_pwd2.value='" + item["password"] + "';") # 输入密码 # pwd_input = wait.until(EC.visibility_of_element_located((By.ID, 'OPassword'))) # pwd_input.send_keys(item["password"]) # 停顿2秒 准备模拟键盘输入 with Ddxoft(pid) as dd_opt: # 输入用户名 driver.execute_script('login_iframe=document.getElementsByName("login-iframe")[0].contentDocument;' 'username_input=login_iframe.getElementById("LoginId");' 'username_input.focus();' 'username_input.value="' + username + '";') # 切换到登录的框架 driver.switch_to.frame("login-iframe") dd_opt.dd_tab() for i in item['password']: dd_opt.dd_keyboard(i) sleep(0.5) # 如果出现验证码 try: # 输入验证码 validation_img = wait.until(EC.visibility_of_element_located((By.ID, 'tokenImg'))) left = validation_img.location['x'] top = validation_img.location['y'] right = validation_img.location['x'] + validation_img.size['width'] bottom = validation_img.location['y'] + validation_img.size['height'] photo_base64 = driver.get_screenshot_as_base64() img_bytes = driver_screenshot_2_bytes(photo_base64, (left, top, right, bottom)) captcha_code = self.ask_image_captcha(img_bytes, username, file_type=".png") token_input = wait.until(EC.visibility_of_element_located((By.ID, 'verCodeToken'))) token_input.send_keys(captcha_code) except CaptchaTimeout: raise except Exception: pass # 点击登录 login_btn = wait.until(EC.visibility_of_element_located((By.ID, 'LoginButton'))) # login_btn.click() driver.execute_script("Javascript: return doSubmit()") # 获取出错信息 try: error_div = wait.until(EC.visibility_of_element_located((By.ID, 'errInfo'))) msg = error_div.text if msg != "": yield from self.error_handle(username, msg, tell_msg=msg) return except TimeoutException: pass # 获取cookie cookies = driver.get_cookies() # 获取余额所在的页面 driver.get(self.balance_url) balance_div = wait.until(EC.visibility_of_element_located((By.XPATH, '//td[@id="CanUseBalanceShow_0"]/div'))) item["balance"] = balance_div.text.strip() # 进入到明细查询页面 yield Request( url=self.prequeryhistory_url, callback=self.parse_prequeryhistory, headers=self.headers, cookies=cookies, meta=meta, dont_filter=True, errback=self.err_callback ) except CaptchaTimeout: yield from self.error_handle(username, "浦发银行---等待验证码超时", tell_msg="等待验证码超时,请刷新页面重试。。") except Exception: yield from self.except_handle(username, "浦发银行---登录数据解析异常") finally: driver.quit()
def parse(self, response): item = response.meta["item"] username = item["username"] driver = self.load_page_by_webdriver( "https://perbank.abchina.com/EbankSite/startup.do", "//input[@id='username']") pid = driver.iedriver.process.pid try: driver.maximize_window() with Ddxoft(pid) as visual_keyboard: driver.execute_script( "username_input=document.getElementById('username');" "username_input.value='%s';" "username_input.focus();" % username) visual_keyboard.dd_tab() sleep(0.1) for key in item["password"]: visual_keyboard.dd_keyboard(key) sleep(1.3) # 这样输入密码比较安全 # driver.execute_script("document.getElementById('userNameForm').pwdField.value='" # + item["password"] + "';") # 检查是否需要输入验证码 if driver.find_element_by_id("code").is_displayed(): with Ddxoft(pid) as visual_keyboard: visual_keyboard.dd_tab() captcha_image = driver.find_element_by_id("vCode") location = captcha_image.location size = captcha_image.size left = location["x"] - 8 top = location["y"] right = left + size["width"] bottom = top + size["height"] photo_base64 = driver.get_screenshot_as_base64() captcha_body = driver_screenshot_2_bytes( photo_base64, (left, top, right, bottom)) captcha_code = self.ask_image_captcha(captcha_body, username, file_type=".png") with Ddxoft(pid) as visual_keyboard: driver.execute_script( "captcha_input=document.getElementById('code');" "captcha_input.value='%s';" "captcha_input.focus();" % captcha_code) visual_keyboard.dd_tab() try: self.wait_xpath(driver, "//span[@class='v-code-error right']", raise_timeout=True, timeout=3) except TimeoutException: yield from self.error_handle(username, "农业银行---图形验证码错误", "图形验证码错误,请刷新页面重试。") return driver.execute_script("document.getElementById('logo').click();") try: self.wait_xpath(driver, "//a[@id='logout_a']", raise_timeout=True, timeout=6) except TimeoutException: tell_msg = driver.find_element_by_id('powerpass_ie_dyn_Msg').text or \ driver.find_element_by_class_name("logon-error").get_attribute("title") yield from self.error_handle( username, "农业银行---登录失败:(username:%s, password:%s) %s" % (username, item["password"], '登录失败'), tell_msg=tell_msg) return self.wait_xpath(driver, "//iframe[@id='contentFrame']") driver.switch_to.frame("contentFrame") self.wait_xpath(driver, "//div[@id='m-paycardcontent']") item['balance'] = driver.find_element_by_id("dnormal").text # 跳转到银行流水界面 onclick_fun_str = "toDetail('%s')" % username self.wait_xpath(driver, '//a[@onclick="%s"]' % onclick_fun_str) driver.switch_to.default_content() driver.execute_script( 'document.getElementById("contentFrame").contentWindow.' + onclick_fun_str + ';') self.wait_xpath(driver, "//iframe[@id='contentFrame']") driver.switch_to.frame("contentFrame") self.wait_xpath(driver, '//input[@id="startDate"]') driver.switch_to.default_content() end_date = date.today() # 有可能失败,执行两次 start_date_str = (end_date - self.date_delta).strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") for i in range(2): driver.execute_script( 'data_iframe=document.getElementById("contentFrame").contentDocument;' 'data_iframe.getElementById("startDate").value="%s";' 'data_iframe.getElementById("endDate").value="%s";' % (start_date_str, end_date_str)) sleep(0.6) driver.execute_script( 'data_iframe.getElementById("btn_query").click();') item["trade_records"] = self.parse_item_from_webpage(driver) yield from self.crawling_done(item) except CaptchaTimeout: yield from self.error_handle(username, "农业银行---等待验证码超时", tell_msg="等待验证码超时,请刷新页面重试。。") except Exception: yield from self.except_handle(username, msg="农业银行---错误信息", tell_msg="银行流水数据爬取失败,请刷新页面重试!") finally: driver.quit()
def parse(self, response): meta = response.meta item = meta["item"] username = item["username"] driver = Ie(executable_path=self.settings['IE_EXECUTABLE_PATH']) pid = driver.iedriver.process.pid try: driver.maximize_window() wait = ui.WebDriverWait(driver, 20) driver.get(response.url) username = wait.until(EC.visibility_of_element_located((By.ID, 'textfield'))) username.send_keys(username) # 停顿2秒 准备模拟键盘输入 with Ddxoft(pid) as visual_keyboard: visual_keyboard.dd_tab() visual_keyboard.dd_tab() for i in item['password']: visual_keyboard.dd_keyboard(i) sleep(0.5) # 输入验证码 validation_img = wait.until(EC.visibility_of_element_located((By.ID, '_tokenImg'))) left = validation_img.location['x'] top = validation_img.location['y'] right = validation_img.location['x'] + validation_img.size['width'] bottom = validation_img.location['y'] + validation_img.size['height'] photo_base64 = driver.get_screenshot_as_base64() img_bytes = driver_screenshot_2_bytes(photo_base64, (left, top, right, bottom)) captcha_code = self.ask_image_captcha(img_bytes, username, file_type=".png") token_input = wait.until(EC.visibility_of_element_located((By.ID, '_vTokenName'))) token_input.send_keys(captcha_code) button = driver.find_element_by_id('button') button.click() # 判断是否登录成功 try: error_info = wait.until(EC.visibility_of_element_located((By.ID, 'EEE'))) msg = error_info.text if msg != "": yield from self.error_handle(username, msg, tell_msg=msg) return except TimeoutException: pass cookies = driver.get_cookies() yield Request( url=self.account_url, callback=self.parse_act_list, headers=self.headers, cookies=cookies, meta=meta, dont_filter=True, errback=self.err_callback ) except CaptchaTimeout: yield from self.error_handle(username, "邮政银行---等待验证码超时", tell_msg="等待验证码超时,请刷新页面重试。。") except Exception: yield from self.except_handle(username, "邮政银行---登录数据解析异常") finally: driver.quit()
def parse(self, response): meta = response.meta item = meta['item'] mail_type = meta['mail_type'] username = item["username"] password = item["password"] url = self._start_url_ driver = self.load_page_by_webdriver(url, '//iframe') try: if mail_type == '163.com': host = 'http://mail.163.com/' iframe = driver.find_element_by_tag_name('iframe') elif mail_type == '126.com': host = 'http://mail.126.com/' tab_126 = driver.find_element_by_class_name('item-126') tab_126.click() driver.implicitly_wait(1) iframe = driver.find_elements_by_tag_name('iframe')[1] elif mail_type == 'yeah.net': host = 'http://mail.yeah.net/' tab_yeah = driver.find_element_by_class_name('item-yeah') tab_yeah.click() driver.implicitly_wait(1) iframe = driver.find_elements_by_tag_name('iframe')[1] else: err_message = '不支持非163、126、yeah邮箱系统' yield from self.error_handle( username, msg="163---不支持此邮箱:(username:%s, password:%s) %s" % (username, password, err_message), tell_msg=err_message) return driver.execute_script( '''document.querySelector("a[data-style='7']").click();''') driver.switch_to.frame(iframe) self.wait_xpath(driver, "//a[@id='dologin']") driver.execute_script( 'document.getElementsByName("email")[0].value="%s";' 'document.getElementsByName("password")[0].value="%s";' 'document.getElementById("dologin").click();' % (username.split('@', 1)[0], password)) driver.implicitly_wait(1) # 切换到表单 try: if driver.find_element_by_id('ScapTcha'): sleep(1) validat_img = False for i in range(self.captcha_retry_time): img_desc = driver.find_element_by_xpath( '//div[@class="ncpt_panel"]' '/div[@class="ncpt_hint_txt"]').text sleep(0.5) img_url = driver.find_element_by_xpath( '//div[@class="ncpt_pzzClick"]' '/img').get_attribute('src') x1, y1, x2, y2, x3, y3 = self.send_pic_get_ponint6_to_ssdb( img_url, img_desc, username) driver.find_element_by_id('ScapTcha').click() e163 = E_3click(driver) e163.click_3_char('//div[@class="ncpt_pzzClick"]/img', x1, y1, x2, y2, x3, y3) driver.execute_script( "document.getElementById('dologin').click();") sleep(1) if "帐号或密码" in driver.find_element_by_class_name( "ferrorhead").text: err_message = '账号或密码错误' yield from self.error_handle( username, msg="163---登录失败:(username:%s, password:%s) %s" % (username, password, err_message), tell_msg=err_message) return if "成功" in driver.find_element_by_xpath( '//div[@class="ncpt_panel"]' '/div[@class="ncpt_hint_txt"]').text: validat_img = True break elif "点击" in driver.find_element_by_xpath('//div[@class="ncpt_panel"]' '/div[@class="ncpt_hint_txt"]').text \ or "失败" in driver.find_element_by_xpath('//div[@class="ncpt_panel"]' '/div[@class="ncpt_hint_txt"]').text: self.wait_xpath( driver, '//div[@class="ncpt_panel"]/div[contains(text(),"点击")]' ) else: validat_img = True break if not validat_img: err_message = "点击验证错误四次,请重试!" yield from self.error_handle( username, msg="163---登录失败:(username:%s, password:%s) %s" % (username, password, err_message), tell_msg=err_message) return except NoSuchElementException: pass try: if 'errorAlert-show' in driver.find_element_by_id( 'errorAlert').get_attribute('class'): err_message = '账号或密码错误' yield from self.error_handle( username, msg="163---登录失败:(username:%s, password:%s) %s" % (username, password, err_message), tell_msg=err_message) return except NoSuchElementException: pass try: if driver.find_element_by_name("phonecode"): click_btn = driver.find_element_by_class_name('getsmscode') click_btn.click() sms_code = self.ask_sms_captcha(username) driver.find_element_by_name("phonecode").send_keys( sms_code) driver.find_element_by_class_name("u-loginbtn").click() sleep(1) except NoSuchElementException: pass curr_url = driver.current_url # self.logger.debug('++++\ncurrent_url :{0}\n+++++'.format(curr_url)) # 跳转多种页面。 if url == curr_url or 'errorType' in curr_url: # 说明未登录成功 err_message = '账号或密码错误' yield from self.error_handle( username, msg="163---登录失败:(username:%s, password:%s) %s" % (username, password, err_message), tell_msg=err_message) return if 'clearlimit' in curr_url: # 说明未登录成功跳转到检查网站 phone = self.ask_extra_captcha(username) # self.logger.debug('输入电话:{0}'.format(phone)) driver.execute_script( "document.getElementById('password').value='{0}';" "document.getElementById('mobile').value='{1}';".format( password, phone)) driver.find_element_by_class_name('yzm_btn').click() driver.implicitly_wait(0.5) try: if driver.find_element_by_id( 'dg-usercheckcode').is_displayed(): # cookies = get_cookies_dict_from_webdriver(driver) # self.logger.debug('cookies:{0}'.format(cookies)) # headers = { # 'Accept': '*/*', # 'Accept-Encoding': 'gzip, deflate', # 'Accept-Language': 'zh-CN,zh;q=0.9', # 'Connection': 'keep-alive', # 'Host': 'reg.163.com', # 'Referer': 'http://reg.163.com/clearlimit/check.jsp?username={0}' # '&url=http%3A%2F%2Fentry.mail.163.com%2Fcoremail%2Ffcg%2Fntesdoor2' # '%3Fdf%3Dwebmail163%26from%3Dweb%26style%3D-1%26product%3Dmail163' # '%26uid%3D{1}'.format(username, url_parse.quote(username)), # 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36', # 'X-Requested-With': 'XMLHttpRequest', # } # get_id = get_content_by_requests('http://reg.163.com/services/getid',headers=headers,cookie_jar=cookies).decode() # self.logger.debug('get_id:{0}'.format(get_id)) # capcha_body = get_content_by_requests('http://reg.163.com/services/getimg?v={0}&num=6&type=2&id={1}'.format(get_js_time(),get_id),cookie_jar=cookies,headers=headers) # self.logger.debug('capcha_body:{0}'.format(capcha_body)) wait = ui.WebDriverWait(driver, 20) validation_img = wait.until( EC.visibility_of_element_located( (By.ID, 'dg-checkCode'))) left = validation_img.location['x'] top = validation_img.location['y'] right = validation_img.location[ 'x'] + validation_img.size['width'] bottom = validation_img.location[ 'y'] + validation_img.size['height'] photo_base64 = driver.get_screenshot_as_base64() capcha_body = driver_screenshot_2_bytes( photo_base64, (left, top, right, bottom)) captcha_code = self.ask_image_captcha( capcha_body, username) driver.find_element_by_id( 'dg-usercheckcode').send_keys(captcha_code) driver.find_element_by_class_name('iDialogBtn').click() driver.implicitly_wait(0.5) try: # self.logger.debug('err?:{0}'.format(driver.find_element_by_id('dg-err') # .get_attribute('class'))) if 'err' in driver.find_element_by_id( 'dg-err').get_attribute('class'): yield from self.error_handle( username, msg="验证码输入错误", tell_msg="验证码输入错误") return # self.logger.debug('p {0}'.format(driver.find_element_by_tag_name('p'))) if driver.find_element_by_tag_name('p'): yield from self.error_handle( username, msg=driver.find_element_by_tag_name( 'p').text, tell_msg=driver.find_element_by_tag_name( 'p').text) return except Exception: alert = driver.switch_to.alert self.logger.debug("Alert:%s" % alert.text) if '请查收' not in alert.text: yield from self.error_handle( username, msg=alert.text, tell_msg=alert.text) return alert.accept() driver.switch_to.default_content() except Exception: pass # driver.get_screenshot_as_file("/logs/phone.jpg") sms_code = self.ask_sms_captcha(username) # self.logger.debug('输入验证码:{0}'.format(sms_code)) # self.logger.debug('page_source:{0}'.format(driver.page_source)) driver.find_element_by_id('mobcheckcode').send_keys(sms_code) # driver.execute_script("document.getElementById('mobcheckcode').value='{0}';".format(sms_code)) driver.find_element_by_id('loginBtn').click() driver.implicitly_wait(2) if len(driver.find_element_by_id('eHint').text) > 6: yield from self.error_handle( username, msg=driver.find_element_by_id('eHint').text, tell_msg=driver.find_element_by_id('eHint').text) return # driver.get_screenshot_as_file("/logs/sms.jpg") driver.implicitly_wait(1) folder_str = self.folders_pattern.search(driver.page_source) if not folder_str: yield from self.error_handle(username, msg="没有文件夹", tell_msg="出现未知异常") return else: folder_str = folder_str.group(1) folder_list = self.folder_id_pattern.findall(folder_str) folder_list.remove('2') # 草稿箱 folder_list.remove('3') # 已发送 folder_list.remove('6') # 病毒邮件 folder_str = ''.join('<int>' + str(x) + '</int>' for x in folder_list) # self.logger.debug('driver_current_url: %s' % driver.current_url) # sid = driver.current_url.split('?', 1)[-1].split('&', 1)[0] sid = parse_qs(urlsplit(driver.current_url).query)["sid"][0] headers = { 'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.8', 'Content-type': 'application/x-www-form-urlencoded', 'Host': 'mail.%s' % mail_type, 'Origin': 'http://mail.%s' % mail_type, 'Referer': 'http://mail.%s/js6/main.jsp?sid=%s&df=mail163_letter' % (mail_type, sid), 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3050.3 Safari/537.36', } meta['sid'] = sid meta['host'] = host cookie = driver.get_cookies() # meta['folder_str'] = folder_str # mbox_url = '{0}js6/main.jsp?sid={1}&df=mail{2}_letter#module=mbox.ListModule' \ # '%7C%7B%22fid%22%3A1%2C%22order%22%3A%22date%22%2C%22' \ # 'desc%22%3Atrue%7D'.format(host, sid, mail_type[mail_type.index(".")]) # yield Request(url=mbox_url, # headers=headers, # cookies=cookie, # meta=meta, # callback=self.parse_mbox, # dont_filter=True, # errback=self.err_callback) search_url = host + 'js6/s?sid=' + sid + '&func=mbox:searchMessages' data = { 'var': '<?xml version="1.0"?><object><array name="conditions"><object>' '<array name="conditions"><object><string name="field">subject</string>' '<string name="operator">contains</string><string name="operand">{0}</string>' '<boolean name="ignoreCase">true</boolean></object></array><string name="operator">or' '</string></object></array><int name="windowSize">{1}</int><string name="order">date' '</string><boolean name="desc">true</boolean><boolean name="returnTag">true</boolean>' '<boolean name="recursive">true</boolean><array name="fids">{2}</array>' '</object>'.format(CREDIT_CARD_KEYWORD, self.PAGE_PER_COUNT, folder_str), } yield FormRequest(url=search_url, callback=self.parse_search, formdata=data, meta=meta, headers=headers, cookies=cookie, dont_filter=True, errback=self.err_callback) except Exception: yield from self.except_handle(username, msg="邮箱登录异常", tell_msg="邮箱登录异常") finally: driver.quit()