def __init__(self):
capabilities = DesiredCapabilities.CHROME
# capabilities["loggingPrefs"] = {"performance": "ALL"} # chromedriver < ~75
capabilities["goog:loggingPrefs"] = {
"performance": "ALL"
} # chromedriver 75+
self.driver = Chrome(CHROMEDRV_PATH, desired_capabilities=capabilities)
def main():
option = None
mysql_db = DataBase()
# 配置文件中开启是否无头，生产阶段关闭
if if_headless():
option = webdriver.ChromeOptions()
option.add_argument(argument='headless')
option.add_argument('--no-sandbox')
browser = Chrome(chrome_options=option)
url_1 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?method=query&querytype=productname&pfid=&content=&dataPage=0&perPage=15&allRows=8084&order='
response = browser.request('post', url_1)
response1 = json.loads(response.content)['pageBean']
page = response1['allPage']
browser.close()
for i in range(0, page):
browser = Chrome(chrome_options=option)
url_1 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?method=query&querytype=productname&pfid=&content=&dataPage=' + str(
i) + '&allPage=539&perPage=15&allRows=8084&order='
res = browser.request('post', url_1)
res1 = json.loads(res.content)['list']
browser.close()
for j in range(len(res1)):
sql = "insert into t_jkftsythzpbaxx_lbsj(c_bh, dt_insertTime, c_url, b_content, c_json,c_page) VALUES (REPLACE(UUID(),\"-\",\"\"), sysdate(), %s,%s,%s,%s)"
mysql_db.exetcute_sql(sql, [
url_1, res.content,
parse(res1[j]),
str(i + 1) + '_' + str(j + 1)
])
def main():
option = None
mysql_db = DataBase()
# 配置文件中开启是否无头，生产阶段关闭
if if_headless():
option = webdriver.ChromeOptions()
option.add_argument(argument='headless')
option.add_argument('--no-sandbox')
for i in range(1, 330):
browser = Chrome(chrome_options=option)
url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList&on=true&page=' + str(
i) + '&pageSize=15&productName=&conditionType=1&applyname=&applysn'
res = browser.request('post', url)
# print(res.text)
res1 = json.loads(res.content)['list']
# print(res1)
browser.close()
for j in range(len(res1)):
sql = "insert into t_hzpscxkhzqy_lbsj(c_bh, dt_insertTime, c_url, b_content, c_json,c_page) VALUES (REPLACE(UUID(),\"-\",\"\"), sysdate(), %s,%s,%s,%s)"
mysql_db.exetcute_sql(
sql,
[url, res.content,
parse(res1[j]),
str(i) + '_' + str(j + 1)])
def _create_web_driver_at_mint_com(
headless=False,
session_path=None,
use_chromedriver_on_path=False,
chromedriver_download_path=os.getcwd(),
):
"""
Handles starting a web driver at mint.com
"""
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument("headless")
chrome_options.add_argument("no-sandbox")
chrome_options.add_argument("disable-dev-shm-usage")
chrome_options.add_argument("disable-gpu")
# chrome_options.add_argument("--window-size=1920x1080")
if session_path is not None:
chrome_options.add_argument("user-data-dir=%s" % session_path)
if use_chromedriver_on_path:
driver = Chrome(options=chrome_options)
else:
driver = Chrome(
options=chrome_options,
executable_path=get_stable_chrome_driver(
chromedriver_download_path),
)
return driver
def creat_zombie(c):
print(f'''Processing: {c}''')
c = val.clean_str(c).replace(' ', '_')
driver = Chrome(driver_path)
# first load a page
driver.get(config.base_url)
# then load the cookie
cookies = pickle.load(open(cookie_path, "rb"))
for cookie in cookies:
driver.add_cookie(cookie)
tars = []
with open(r'{}/pages_{}.json'.format(config.path_data, c), 'r') as f:
urls = json.loads(f.read())
for u in urls:
driver.get(u)
time.sleep(config.sleep_long)
soup = BeautifulSoup(driver.page_source, 'html5lib')
# imgs = soup.findAll( 'img', class_='ImageThumbnail-image' )
imgs = soup.findAll('img', class_='info-card-media-user')
if (len(imgs) == 0):
l.commit(type='error', msg=f'Failed! Country: {c}, url: {u}')
break
# store the image urls
for e in imgs:
tars.append(e['src'])
# commit the result to file
rw.write_to_json(
r'{}/{}'.format(config.path_data, 'imgs_{}.json'.format(c)), tars)
# safely quit the driver
driver.quit()
def query_engr_registration(registration_no, driver: Chrome):
try:
url1 = 'https://portal.bpeq.qld.gov.au/BPEQPortal/Search_for_a_RPEQ/BPEQPortal/Engineer_Search.aspx'
driver.get(url1)
element = WebDriverWait(driver, 16).until(EC.presence_of_element_located(
(By.ID, 'ctl01_TemplateBody_WebPartManager1_gwpciEngineersearch_ciEngineersearch_ResultsGrid_Sheet0_Input3_TextBox1')))
element.send_keys(registration_no)
element = WebDriverWait(driver, 16).until(EC.presence_of_element_located(
(By.ID, 'ctl01_TemplateBody_WebPartManager1_gwpciEngineersearch_ciEngineersearch_ResultsGrid_Sheet0_SubmitButton')))
element.click()
element = WebDriverWait(driver, 16).until(EC.presence_of_element_located(
(By.XPATH, '//*[@id="ctl01_TemplateBody_WebPartManager1_gwpciEngineersearch_ciEngineersearch_ResultsGrid_Grid1_ctl00__0"]/td[1]/a')))
registration_no = element.get_attribute('href').split("=")[1]
url = f'https://portal.bpeq.qld.gov.au/Party.aspx?ID={registration_no}'
response = driver.request("GET", url, verify=False)
soup = BeautifulSoup(response.text, 'html.parser')
parts = [p.text.strip("\r\n\t ")
for p in soup.select('.PanelFieldValue > span')]
print(f"Num Parts: {len(parts)}")
if len(parts) == 12:
name = parts[0]
company = parts[1]
date_joined = parts[2]
job_type = parts[3]
status = parts[4]
date_registered = parts[5]
return name, company, date_joined, job_type, status, date_registered
elif len(parts) == 11:
name = parts[0]
date_joined = parts[1]
job_type = parts[2]
status = parts[3]
date_registered = parts[4]
return name, None, date_joined, job_type, status, date_registered
elif len(parts) == 16:
name = parts[0]
company = parts[1]
date_joined = parts[2]
job_type = parts[3]
_ = parts[4]
status = parts[5]
date_registered = parts[6]
return name, company, date_joined, job_type, status, date_registered
elif len(parts) == 15:
name = parts[0]
date_joined = parts[1]
job_type = parts[2]
_ = parts[3]
status = parts[4]
date_registered = parts[5]
return name, None, date_joined, job_type, status, date_registered
except Exception as e:
print(e)
def main():
# file_reader = open(IMAGE_URLS_FILE)
# try:
# print(list(file_reader))
# finally:
# file_reader.close()
# return
all_links = []
with open(IMAGE_URLS_FILE, 'r') as reader:
# Read and print the entire file line by line
line = reader.readline()
while line != '': # The EOF char is an empty string
# print(line, end='')
if line[0] != '-':
all_links.append(line.strip())
line = reader.readline()
webdriver = Chrome()
for url in all_links:
try:
filename = re.search(r"(boohoo\/.+\?)", url).group()
filename = filename[7:-1]
filename = IMGFOLDER + re.sub("[/%]", "_", filename) + '.jpeg'
print(filename)
response = webdriver.request('GET', url)
file = open(filename, 'wb')
for chunk in response.iter_content(10000):
file.write(chunk)
file.close()
finally:
continue
webdriver.quit()
def __init__(self, user_phone, user_password, cookies=None):
self._user_phone = user_phone
self._user_password = user_password
self._is_authorized = False
self._captcha = None # REMOVE
options = Options()
options.add_argument("--headless")
WINDOW_SIZE = "1620,880" # ?
options.add_argument("--window-size=%s" % WINDOW_SIZE)
self.webdriver = Chrome(chrome_options=options)
if cookies:
self.webdriver.get('https://messages.megafon.ru/onebox/mix.do')
for cookie in cookies:
self.webdriver.add_cookie(cookie)
self.webdriver.get('https://messages.megafon.ru/onebox/mix.do')
self._is_authorized = self.try_is_authorized()
if not self.is_authorized:
self.webdriver.get(r"https://messages.megafon.ru/onebox/mix.do")
phone_input = self.webdriver.find_element_by_id('user12')
phone_input.send_keys(user_phone)
password_input_secinfo = self.webdriver.find_element_by_id('secinfo')
password_input_secinfo.click()
password_input = self.webdriver.find_element_by_id("secinfo2")
password_input.send_keys(user_password)
def __init__(self, current_user):
from selenium import webdriver
from seleniumrequests import Chrome
self.current_user = current_user
self.headers = {
"Authority":
"www.supremenewyork.com",
"Method":
"GET",
"Path":
"/",
"Scheme":
"https",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding":
"gzip,deflate,br",
"Accept-Language":
"en-US,en;q=0.9,ru;q=0.8",
"Cache-Control":
"max-age=0",
"Upgrade-Insecure-Requests":
"1",
"User-Agent":
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)"
}
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
self.browser = Chrome(os.path.dirname(os.path.realpath(__file__)) +
'/chromedriver',
chrome_options=chrome_options)
def get_driver(url, login, password):
chrome_options = get_chrome_options()
driver = Chrome(chrome_options=chrome_options)
driver.get(url)
gdpr_click(driver)
log_in(driver, login, password)
return driver
def login(user, passwd):
# Type User and password into site. Incorrect username = unclickable button = timeout. Incorrect password = no token
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_experimental_option('excludeSwitches',
['enable-logging'])
driver = Chrome(
"C:\\bin\\chromedriver",
options=chrome_options) # TODO: Add var for webdriver location
driver.get(login_url)
un_field = (By.ID, "i0116")
pw_field = (By.ID, "i0118")
next_button = (By.ID, "idSIButton9")
WebDriverWait(driver, 20).until(
ec.presence_of_element_located(un_field)).send_keys(user)
WebDriverWait(driver,
20).until(ec.element_to_be_clickable(next_button)).click()
WebDriverWait(driver, 20).until(
ec.element_to_be_clickable(pw_field)).send_keys(passwd)
log_in = (By.XPATH, '//*[contains(@value, "Sign in")]')
WebDriverWait(driver,
20).until(ec.presence_of_element_located(log_in)).click()
return driver
def __init__(self, url, username, password):
self.bearer = None
# enable browser logging
d = DesiredCapabilities.CHROME
d['loggingPrefs'] = {'browser': 'ALL'}
self.server = Server("BrowserMob\\bin\\browsermob-proxy")
cli.print_info("Starting proxy server...")
self.server.start()
self.proxy = self.server.create_proxy()
cli.add_to_print("OK\n\n")
cli.print_warning("NOTE: Connections will show as INSECURE!\n")
options = webdriver.ChromeOptions()
self.url = url
options.add_argument("--proxy-server={0}".format(self.proxy.proxy))
cli.print_info("Browser started\n")
self.browser = Chrome(options=options,
executable_path="./chromedriver.exe",
desired_capabilities=d)
self.username = username
self.password = password
del password
del username
def __init__(self, user, password):
self.options = Options()
self.driver = Chrome(
executable_path=os.path.abspath("/usr/bin/chromedriver"),
options=self.options)
self.login(user, password)
self.dissertations = self.get_list_of_dissertations()
self.lookup_values = self.__review_dissertations()
self.__lookup_decisions()
def runReport(projects, start_date='', end_date=''):
driver = Chrome(CHROMEDRIVER_PATH
) # Optional argument, if not specified will search path.
#driver.set_window_size(300,200)
runReportRequest = getRunReportRequest(projects, start_date, end_date)
driver.get(runReportRequest)
return driver
def get_web_driver(url, username, password):
driver = Chrome()
driver.get(url)
driver.implicitly_wait(1) # seconds
driver.find_element_by_id("ctl00_Main_LoginBox_UserName").send_keys(
username)
driver.find_element_by_id("ctl00_Main_LoginBox_Password").send_keys(
password)
return driver
class Browser:
driver = None
def __init__(self):
capabilities = DesiredCapabilities.CHROME
# capabilities["loggingPrefs"] = {"performance": "ALL"} # chromedriver < ~75
capabilities["goog:loggingPrefs"] = {
"performance": "ALL"
} # chromedriver 75+
self.driver = Chrome(CHROMEDRV_PATH, desired_capabilities=capabilities)
def goto(self, url):
self.driver.get(url)
def set_input_value(self, name, value):
self.driver.find_element_by_id(name).send_keys(value)
def click_link(self, class_name):
self.driver.find_element_by_class_name(class_name).click()
def check_elem_exist(self, name):
elem = None
try:
elem = self.driver.find_element_by_xpath(f"//*[text()='{name}']")
finally:
return elem
def quit(self):
self.driver.quit()
def create_webdriver(self):
#https://stackoverflow.com/questions/29916054/change-user-agent-for-selenium-driver
opts = Options()
self.useragent = random_user_agent() #calls for a random useragent
opts.add_argument("user-agent=" +
self.useragent) #adds random user agent to options
self.webdriver = Chrome(
executable_path=get_local_directory() + '/resources/chromedriver',
chrome_options=opts) #creates new webdriver with premade options
self.webdriver.set_page_load_timeout(int(
self.settings["timeout"])) #set timeout of pageload from config
def setupSelenium(self):
CHROMEDRIVER_PATH = '/app/.chromedriver/bin/chromedriver'
chrome_options = Options()
chrome_options.binary_location = os.environ.get('GOOGLE_CHROME_SHIM')
self.driver = Chrome(executable_path=CHROMEDRIVER_PATH, options=chrome_options)
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--window-size=1920,1080")
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument(
'--user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36"')
def _create_browser_with_extension(self):
# 加载本地插件
# self.browser = webdriver.PhantomJS()
# self.browser = PhantomJS()
# self.browser.set_window_size(1120, 550)
chrome_options = Options()
chrome_options.add_extension(adblock_plus_path())
driver = Chrome(chrome_options=chrome_options)
driver.start_client()
return driver
def web_test():
url = conn.server_data.server + conn.server_data.REPORT_TOKEN_PARAMETER
headers = conn.server_data.json_type_headers
data = {}
data['token']= '02cda594401840ec955c81533bc761d4'
path = 'C:\gs26\client\conn\chromedriver.exe'
browser = Chrome(path)
r = browser.request('POST', url, headers=headers, data=json.dumps(data))
print(r.cookies)
browser.get(url)
def web_open():
url = conn.server_data.server + conn.server_data.REPORT_TOKEN_PARAMETER
headers = conn.server_data.json_type_headers
data={}
data['token']=conn.server_data.USER_TOKEN
path_token = os.getcwd()
path = path_token + '\\conn\\chromedriver.exe'
browser = Chrome(path)
res = browser.request('POST', url, headers=headers, data=json.dumps(data))
print(url)
print(res.cookies)
browser.get(url)
def login():
driver = Chrome('/Users/liuxian/Downloads/chromedriver')
driver.get("https://www.tianyancha.com/")
driver.maximize_window()
# 点击登录链接
# loginLink = WebDriverWait(driver, 30).until(lambda x:x.find_element_by_xpath('//a[@onclick="header.loginLink(event)"]'))
loginLink = WebDriverWait(
driver,
30).until(lambda x: x.find_element_by_xpath('//a[text()="登录/注册"]'))
print(loginLink.text)
loginLink.click()
# 切换到密码登录方式
# login_by_pwd = WebDriverWait(driver, 30).until(lambda x:x.find_element_by_xpath('//div[@onclick="loginObj.changeCurrent(1);"]'))
login_by_pwd = WebDriverWait(
driver,
30).until(lambda x: x.find_element_by_xpath('//div[text()="密码登录"]'))
print(login_by_pwd.text)
login_by_pwd.click()
# 输入用户名
username = WebDriverWait(driver, 30).until(
lambda x: x.find_elements_by_xpath('//input[@type="text"]')[4])
username.send_keys('13926278814')
# username.send_keys('13261731251')
# 输入密码
password = WebDriverWait(
driver, 30).until(lambda x: x.find_element_by_css_selector(
'div.input-warp.-block > input.input.contactword'))
password.send_keys('liubing0220')
# password.send_keys('lx109718')
# 取消登录保留一周的选项
# saveoneweek = WebDriverWait(driver, 30).until(lambda x: x.find_element_by_css_selector('input.contactautoLogin'))
# saveoneweek.click()
# 点击登录按钮
# login_button = WebDriverWait(driver, 30).until(lambda x:x.find_element_by_xpath('//div[@onclick="loginObj.loginByPhone(event);"]'))
login_button = WebDriverWait(
driver,
30).until(lambda x: x.find_element_by_xpath('//div[text()="登录"]'))
print(login_button.text)
driver.execute_script("arguments[0].click();", login_button)
print('开始自动登录...')
time.sleep(15)
print('登录完成。')
open(driver, detail_url)
print('请手动通过验证码...')
time.sleep(20)
response = driver.request(
'GET', 'https://dis.tianyancha.com/dis/getInfoById/3027736699.json?')
print(response)
cookie = get_cookie(driver)
return cookie
def query():
tf_user = request.form.get("tf_user")
tf_pwd = request.form.get("tf_pwd")
orgID = request.form.get("orgID")
options = {}
options["memberListCheck"] = request.form.get("memberListCheck")
options["memberGroupsCheck"] = request.form.get("memberGroupsCheck")
options["genreListCheck"] = request.form.get("genreListCheck")
options["sourceCheck"] = request.form.get("sourceCheck")
options["promoCheck"] = request.form.get("promoCheck")
options["sourceStart"] = request.form.get("sourceStart")
options["sourceEnd"] = request.form.get("sourceEnd")
options["promoStart"] = request.form.get("promoStart")
options["promoEnd"] = request.form.get("promoEnd")
time = datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
new_request = Request(orgID=orgID, username=tf_user, status="Processing", time=time)
# form validation - somple
if not options["memberListCheck"] and not options["memberGroupsCheck"] and not options["genreListCheck"] and not options["sourceCheck"] and not options["promoCheck"]:
return apology("Select at least one report")
if options["sourceCheck"]:
if not options["sourceStart"] or not options["sourceEnd"]:
return apology("Date box unfilled")
if options["promoCheck"]:
if not options["promoStart"] or not options["promoEnd"]:
return apology("Date box unfilled")
# form validation - web stuff
chrome_options = Options()
chrome_options.add_argument("--headless")
driver = Chrome(chrome_options=chrome_options, executable_path=app.config["CHROMEDRIVER_URI"])
login_result = tf_login(driver, tf_user, tf_pwd)
if not login_result:
return apology("Incorrect username / password")
org_result = org_exists(driver, orgID)
if not org_result:
return apology("Invalid Organization ID")
driver.quit()
# process request
db.session.add(new_request)
db.session.commit()
db.session.refresh(new_request)
task = processing.delay(orgID, tf_user, tf_pwd, options, new_request.id)
flash('Queued!')
return render_template("main.html", tf_user=tf_user, tf_pwd=tf_pwd, memberListCheck = options["memberListCheck"], memberGroupsCheck = options["memberGroupsCheck"], genreListCheck=options["genreListCheck"], sourceCheck=options["sourceCheck"], promoCheck=options["promoCheck"], sourceStart=options["sourceStart"], sourceEnd=options["sourceEnd"], promoStart=options["promoStart"], promoEnd=options["promoEnd"])
def __init__(self):
self.driver = Chrome(ChromeDriverManager().install())
self.url = 'http://93.188.161.218/'
self.user = ''
self.password = ''
self.arr_dados_planilha = []
#False apenas para gerar o .exe
if (False):
self.caminho_planilha_exe = 'planilha_insert.xls'
else:
self.caminho_planilha_exe = '../../planilha_insert.xls'
self.ler_planilha()
def goTo(self):
'''
open a new browser and navigate to the url given.
'''
browser = Chrome(executable_path=self.path) #create browser object
info = browser.request(
'GET', self.url) #navigate to the website and get site metadata
browser.get(self.url) #reset the webpage
html = browser.page_source #get the html from the current page
soup = BeautifulSoup("".join(html)) #pass the html to the BS parser
self.html, self.info, self.soup, self.browser = (
html, info, soup, browser) #return these objects as attributes
def setup_driver(self, headless: bool = True) -> Chrome:
"""Establish chrome options for, and instantiate, the driver"""
ua = UserAgent().chrome
chrome_options = ChromeOptions()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-setuid-sandbox")
chrome_options.add_argument(f'--user-agent={ua}')
if headless: chrome_options.add_argument('headless')
driver = Chrome(options=chrome_options)
driver.set_window_size(1920, 1080)
return driver
def get_web_driver(url, username, password):
driver = Chrome()
driver.get(url)
driver.implicitly_wait(1) # seconds
driver.find_element_by_id("loginuser").send_keys(username)
driver.find_element_by_id("loginpass").send_keys(password)
return driver
def openDriver():
global driver
options = Options()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--incognito')
driver = Chrome(chrome_options=options)
def RespCode(domain):
webdriver = Chrome()
domain = "http://www." + str(domain)
response = webdriver.request('GET', domain)
response = str(response)[11:][:-2]
webdriver.close()
return response
def get_web_driver(url, username, password):
driver = Chrome()
driver.get(url)
driver.implicitly_wait(5) # seconds
driver.find_element_by_name("username").send_keys(username)
driver.find_element_by_name("password").send_keys(password)
time.sleep(10)
return driver
def get_web_driver(email, password, headless=False, mfa_method=None,
mfa_input_callback=None, wait_for_sync=True,
session_path=None, imap_account=None, imap_password=None,
imap_server=None, imap_folder="INBOX"):
if headless and mfa_method is None:
warnings.warn("Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms")
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd() + os.path.sep + 'chromedriver'
if _platform in ['win32', 'win64']:
executable_path += '.exe'
zip_type = CHROME_ZIP_TYPES.get(_platform)
if not os.path.exists(executable_path):
zip_file_url = CHROME_DRIVER_BASE_URL % (CHROME_DRIVER_VERSION, zip_type)
request = requests.get(zip_file_url)
if request.status_code != 200:
raise RuntimeError('Error finding chromedriver at %r, status = %d' %
(zip_file_url, request.status_code))
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
if session_path is not None:
chrome_options.add_argument("user-data-dir=%s" % session_path)
driver = Chrome(chrome_options=chrome_options, executable_path="%s" % executable_path)
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
try:
element = driver.find_element_by_link_text("Log In")
except NoSuchElementException:
# when user has cookies, a slightly different front page appears
driver.implicitly_wait(0) # seconds
element = driver.find_element_by_link_text("LOG IN")
driver.implicitly_wait(20) # seconds
element.click()
time.sleep(1)
email_input = driver.find_element_by_id("ius-userid")
# It's possible that the user clicked "remember me" at some point, causing
# the email to already be present. If anything is in the input, clear it
# and use the provided email, just to be safe.
# email_input.setAttribute("value", "")
email_input.clear()
email_input.send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
# An implicitly_wait is also necessary here to avoid getting stuck on
# find_element_by_id while the page is still in transition.
driver.implicitly_wait(1)
time.sleep(1)
# bypass "Let's add your current mobile number" interstitial page
try:
skip_for_now = driver.find_element_by_id('ius-verified-user-update-btn-skip')
skip_for_now.click()
except (NoSuchElementException, StaleElementReferenceException, ElementNotVisibleException):
pass
driver.implicitly_wait(1) # seconds
try:
driver.find_element_by_id('ius-mfa-options-form')
try:
mfa_method_option = driver.find_element_by_id('ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id("ius-mfa-options-submit-btn")
mfa_method_submit.click()
if mfa_method == 'email' and imap_account:
mfa_code = get_email_code(imap_account, imap_password, imap_server, imap_folder=imap_folder)
else:
mfa_code = (mfa_input_callback or input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id("ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id("ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except Exception: # if anything goes wrong for any reason, give up on MFA
mfa_method = None
warnings.warn("Giving up on handling MFA. Please complete "
"the MFA process manually in the browser.")
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
try:
# Status message might not be present straight away. Seems to be due
# to dynamic content (client side rendering).
status_message = WebDriverWait(driver, 30).until(
expected_conditions.visibility_of_element_located(
(By.CSS_SELECTOR, ".SummaryView .message")))
WebDriverWait(driver, 5 * 60).until(
lambda x: "Account refresh complete" in status_message.get_attribute('innerHTML')
)
except (TimeoutException, StaleElementReferenceException):
warnings.warn("Mint sync apparently incomplete after 5 minutes. Data "
"retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
def get_web_driver(email, password):
driver = Chrome()
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
driver.find_element_by_link_text("Log In").click()
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(1)
# Wait until the overview page has actually loaded.
driver.implicitly_wait(20) # seconds
driver.find_element_by_id("transaction")
return driver
