def get_cookie(username, password, proxy):
"""
Launch a chrome to get cookies
"""
chromeopts = ChromeOptions()
if proxy:
chromeopts.add_argument('--proxy-server=%s' % proxy)
web = Chrome(chrome_options=chromeopts)
try:
return signin(web, username, password)
finally:
web.quit()
def setUpClass(cls):
super(SeleniumTestCase, cls).setUpClass()
print('Initializing browser engine...')
if sys.platform == 'win32':
# Chrome hangs up on Windows
capabilities = DesiredCapabilities.FIREFOX
capabilities['loggingPrefs'] = {'browser': 'ALL'}
cls.browser = Firefox(capabilities=capabilities)
else:
capabilities = DesiredCapabilities.CHROME
capabilities['loggingPrefs'] = {'browser': 'ALL'}
options = ChromeOptions()
options.add_argument('headless')
options.add_argument('disable-gpu')
cls.browser = Chrome(chrome_options=options,
desired_capabilities=capabilities)
print('Browser engine initialized.')
def setUp(self, browser):
self.browser = browser
if "firefox" in self.browser:
profile = FirefoxProfile()
# profile.set_preference("plugin.state.silverlight", 2)
# profile.set_preference("browser.download.folderList", 1)
# profile.set_preference("pdfjs.disabled", False);
# profile.set_preference("pdfjs.firstRun", True);
self.driver = Firefox(profile) # get a new firefox session
if "chrome" in self.browser:
chromedriver = "/usr/local/bin/chromedriver"
options = ChromeOptions()
options.add_experimental_option('excludeSwitches', ['disable-component-update'])
options.add_argument("--user-data-dir=./browser_resources/chrome_data_dir/")
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = Chrome(executable_path=chromedriver, chrome_options=options)
self.home_page = home.Home(self.driver)
def create_download_dir_capabilities_for_chrome(path_to_download, **extensions_files):
"""
Example use
| ${capabilities} | create_download_dir_capabilities_for_chrome | Artifacts |
| Open Browser Extension | https://support.spatialkey.com/spatialkey-sample-csv-data/ | gc | desired_capabilities=${capabilities} |
| Click Element | //a[contains(@href,'sample.csv.zip')] |
"""
path_to_download_check = validate_create_artifacts_dir(path_to_download)
chrome_options = ChromeOptions()
prefs = {"download.default_directory": path_to_download_check, "directory_upgrade": "true"}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("--disable-web-security")
for single_extension in extensions_files:
chrome_options.add_extension(single_extension)
logger.info("Chrome Capabilities set download dir '" + path_to_download_check + "'")
return chrome_options.to_capabilities()
def selenium_browser():
# type: () -> Chrome
options = ChromeOptions()
options.add_argument("headless")
options.add_argument("no-sandbox")
options.add_argument("window-size=1920,1080")
return Chrome(options=options)
def login(account, passwd, url):
# 如果driver没加入环境变量中，那么就需要明确指定其路径
# 验证于2017年4月11日
# 直接登陆新浪微博
chrome_options = ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
driver = webdriver.Chrome('/root/qk_python/python/data/collect/weibo_spider/priv/chromedriver',
chrome_options=chrome_options)
driver.maximize_window()
driver.set_page_load_timeout(30)
driver.set_window_size(1124, 850)
# locator = (By.)
driver.get(url)
print('开始登陆')
name_field = driver.find_element_by_id('loginname')
name_field.clear()
name_field.send_keys(account)
password_field = driver.find_element_by_class_name('password').find_element_by_name('password')
password_field.clear()
password_field.send_keys(passwd)
submit = driver.find_element_by_xpath('//*[@id="pl_login_form"]/div/div[3]/div[6]/a/span')
ActionChains(driver).double_click(submit).perform()
time.sleep(5)
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'WB_miniblog')))
source = driver.page_source
if is_login(source):
print('登录成功')
sina_cookies = driver.get_cookies()
driver.quit()
return sina_cookies
def setup_for_test(self, test):
chrome_options = ChromeOptions()
chrome_options.add_argument("test-type")
chrome_options.add_argument("disable-infobars")
chrome_options.add_experimental_option('prefs', {
'credentials_enable_service': False,
'profile.password_manager_enabled': False,
'profile.default_content_setting_values.plugins': 1,
'profile.content_settings.plugin_whitelist.adobe-flash-player': 1,
'profile.content_settings.exceptions.plugins.*,*.per_resource.adobe-flash-player': 1
})
if test.use_proxy:
chrome_options.add_argument("--proxy-server={0}".format(test.proxy_address))
self.capabilities = chrome_options.to_capabilities()
logger.debug("Chrome capabilities: {}".format(self.capabilities))
def set_chrome_options(self):
from selenium.webdriver import ChromeOptions
opts = ChromeOptions()
### Add Boolean Arguments
if T.has_key('true_opts'):
for it in T['true_opts']:
opts.add_argument( '%s=1' % it )
if T.has_key('false_opts'):
for it in T['false_opts']:
opts.add_argument( '%s=0' % it )
value_opts = [
'profile-directory',
'log-level', # 0 to 3: INFO = 0, WARNING = 1, LOG_ERROR = 2, LOG_FATAL = 3
'net-log-capture-mode', # "Default" "IncludeCookiesAndCredentials" "IncludeSocketBytes"'
'register-font-files', # might be windows only
'remote-debugging-port',
'user-agent',
'user-data-dir', # don't use b/c it negates no-extension options
]
### Add Value Arguments
for it in value_opts:
if T.has_key(it):
opts.add_argument( '%s=%s' % (it,T[it]) )
### OTHER CHROME OPTIONS NOT YET FULLY CONFIGURED
# -extensions list str
# -localState dict
# -prefs dict
# set_profile()
# -detach bool
# -debuggerAddress str
# -excludeSwitches list str
# -minidumpPath str
# -mobileEmulation dict
# -perfLoggingPrefs OBJECT (dict)
# set_performance_logging()
return opts
def setup_browser():
if use_firefox:
world.browser = MyFirefox()
world.browser.set_window_size(450, 1200)
world.browser.set_window_position(0, 0)
#world.browser.maximize_window()
elif use_phantomjs:
world.browser = MyPhantomJS()
elif use_headless_chrome:
options = ChromeOptions()
options.add_argument("--window-size=1005,9999")
options.add_argument("--headless");
world.browser = MyChrome(executable_path=os.path.join('..', '..', 'chromedriver'), chrome_options=options)
else:
options = ChromeOptions()
options.add_argument("--start-maximized");
world.browser = MyChrome(executable_path=os.path.join('..', '..', 'chromedriver'), chrome_options=options)
world.da_path = default_path
world.wait_seconds = default_wait_seconds
from selenium.webdriver import Chrome, ChromeOptions
import time
import pymysql
import sys
import spiderByUrl_zsf
option = ChromeOptions() # 创建配置实例
option.add_argument('--headless') #在后台启动
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-shm-usage')
option.add_argument('blink-settings=imagesEnabled=false')
option.add_argument('--disable-gpu')
#链接数据库遍历所有表
conn = pymysql.connect(host="39.106.96.175",
port=3306,
db="scholar_info",
user="******",
password="******",
charset="utf8")
cls = conn.cursor()
sql1 = "select table_name from information_schema.tables where table_schema='scholar_info'" # 获取当前所有表（学校）
cls.execute(sql1)
conn.commit()
#results = cls.fetchall()
#从输入的表名（学校名）和id号开启搜索
result = sys.argv[1]
dataid = sys.argv[2]
results = ['%s' % result]
totnum = 0
havenum = 0
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import string
from browsermobproxy import Server
from BmpProxy import ProxyManager
import json
import time
import requests
proxy = ProxyManager()
server = proxy.start_server()
client = proxy.start_client()
options = ChromeOptions()
options.add_argument("--proxy-server={0}".format(client.proxy))
caps = DesiredCapabilities.CHROME
caps['goog:loggingPrefs'] = {'performance': 'ALL'}
driver = Chrome(options=options, desired_capabilities=caps)
driver.get(
"https://www.linkedin.com/uas/login?session_redirect=%2Fsales&fromSignIn=true&trk=navigator"
)
browser_log = driver.get_log('performance')
username = driver.find_element(
By.ID, "username").send_keys("*****@*****.**")
password = driver.find_element(By.ID,
"password").send_keys("Odette123" + Keys.ENTER)
@Time: 2022-02-28 15:54
@Last_update: 2022-02-28 15:54
@Desc: None
@==============================================@
@ _____ _ _ _ _ @
@ | __|___ ___ ___| |_|_|___| |_| | @
@ | __| _| -_|_ -| | | -_| | . | @
@ |__| |_| |___|___|_|_|_|___|_|___| @
@ Freshield @
@==============================================@
"""
from selenium import webdriver
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
# 设置options参数，以开发者模式运行
option = ChromeOptions()
# 解决报错，设置无界面运行
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-shm-usage')
option.add_argument('blink-settings=imagesEnabled=false') # 不加载图片, 提升速度
option.add_argument('--headless')
user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.50 Safari/537.36'
option.add_argument('user-agent={0}'.format(user_agent))
with webdriver.Remote(
command_executor="http://127.0.0.1:4444/wd/hub",
desired_capabilities=DesiredCapabilities.CHROME, options=option) as driver:
driver.set_window_size(2500, 1200)
driver.get("http://www.baidu.com")
print(driver.title)
def get_driver():
options = ChromeOptions()
options.binary_location = GOOGLE_CHROME_PATH
options.add_argument('--headless')
options.add_argument("start-maximized") # open Browser in maximized mode
options.add_argument("disable-infobars") # disabling infobars
options.add_argument("--disable-extensions") # disabling extensions
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage") # overcome limited resource problem
driver = Chrome(executable_path=DRIVER_PATH, chrome_options=options)
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.implicitly_wait(10)
return driver
import selenium
from selenium import webdriver
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from datetime import date, timedelta
import pdb
options = ChromeOptions()
options.add_argument("--start-maximized")
driver = webdriver.Chrome("./chromedriver", options=options)
start_url = 'http://mrcc.isws.illinois.edu/CLIMATE/'
driver.get(start_url)
try:
WebDriverWait(driver, 600).until(EC.presence_of_element_located((By.NAME, 'textName')))
email_field = driver.find_element_by_name('textName')
email_field.clear()
email_field.send_keys('*****@*****.**')
pwd_field = driver.find_element_by_name('textPwd')
pwd_field.clear()
pwd_field.send_keys('windrose')
driver.find_element_by_name('Submit').click()
try:
WebDriverWait(driver, 600).until(EC.presence_of_element_located((By.NAME, 'chStn')))
def setUp(self):
options = ChromeOptions()
options.add_argument("--start-maximized")
options.add_experimental_option("detach", True)
self.driver = webdriver.Chrome(options=options)
self.driver.get("http://kirv-ui-staging.herokuapp.com/signin")
from selenium.webdriver import Chrome, ChromeOptions
options = ChromeOptions()
options.add_argument('-headless')
browser = Chrome('E:/Python_2/chromedriver')
# 적당한 웹 페이지 열기
browser.get("https://google.com")
# 자바스크립트 실행
r = browser.execute_script("return 100 + 50")
print(r)
options = ChromeOptions()
arguments = [
"--headless",
"--no-sandbox",
"--no-referrers",
"--start-maximized",
"--ignore-certificate-errors",
"--disable-breakpad", "--disable-client-side-phishing-detection", "--disable-session-crashed-bubble",
"--disable-cast-streaming-hw-encoding", "--disable-cloud-import", "--disable-popup-blocking",
"--disable-ipv6",
"--disable-impl-side-painting", "--disable-setuid-sandbox", "--disable-seccomp-filter-sandbox",
"--disable-cast",
"--allow-http-screen-capture"
]
for argument in arguments:
options.add_argument(argument)
chrome_for_google = Chrome(
executable_path='/home/suroegin/chromedriver',
options=options
)
chrome_for_yandex = Chrome(
executable_path='/home/suroegin/chromedriver',
options=options
)
def check(self, campus, term, course):
# check if the course name is valid
# the course will have the form major course_num garbage
# major will be any characters except for numbers
# coures_num can be anything, but it has to start with a number
# garbage can be anything
major = ""
course_num = ""
found = False
for i in range(1, len(course)):
if course[i].isdigit() and course[i - 1] == ' ':
major = course[0:i - 1]
next_space_idx = course.find(' ', i)
temp = course.find('\t', i)
if temp != -1 and temp < next_space_idx:
next_space_idx = temp
if next_space_idx == -1:
# print('Yo')
course_num = course[i:]
else:
# print(course)
# print(next_space_idx)
# print(i)
course_num = course[i:next_space_idx]
found = True
break
if not found:
return CourseStatus.INVALID_COURSE_NAME
# print("Major: " + major)
# print("Course num: " + course_num)
# technical constants
campus_id = "id_campus"
term_name = "term"
major_name = "dept"
search_button_id = "Submit"
first_campus_choice_id = "id_DA"
# load the driver
chrome_options = ChromeOptions()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
driver = webdriver.Chrome(options=chrome_options,
executable_path=PATH_TO_CHROME_DRIVER)
# go to the class search web
driver.get(self.lookup_url)
# click on random campus to show the hidden stuff
script = "document.getElementById('{}').click()".format(
first_campus_choice_id)
driver.execute_script(script)
# select campus
script = "document.getElementById('{}').setAttribute('value', '{}')".format(
campus_id, campus)
driver.execute_script(script)
# select term
term_elem = driver.find_element_by_name(term_name)
for option in term_elem.find_elements_by_tag_name("option"):
if option.text == term:
option.click()
break
# select major
found = False
major_elem = driver.find_element_by_name(major_name)
for option in major_elem.find_elements_by_tag_name("option"):
if option.text == major:
found = True
option.click()
break
if not found:
return CourseStatus.INVALID_MAJOR
# get to the page of all courses in the major offered in the defined semester
search_button = driver.find_element_by_id(search_button_id)
search_button.click()
# the table containing all the courses
table = driver.find_element_by_class_name("zebra-table")
# checking if the courses are offered
offered_right_courses = []
# used to count the number of open right course
count = 0
for row in table.find_elements_by_tag_name("tr"):
info_list = row.find_elements_by_tag_name("td")
if len(info_list) < 17:
continue
#print(info_list[3].text + " " + info_list[0].text + " " + str(info_list[3].text == course_num) + " " + course_num)
if info_list[3].text != course_num:
continue
offered_right_courses.append(row)
count += 1
if count == 0:
return CourseStatus.NOT_OFFERED
# check if it is closed
open_right_courses = []
# used to count the number of open courses
count = 0
for row in offered_right_courses:
info_list = row.find_elements_by_tag_name("td")
if info_list[0].text == "Closed":
continue
open_right_courses.append(row)
count += 1
if count == 0:
return CourseStatus.CLOSED
# check if the course is full
open_right_not_full_courses = []
# used to count the number of open, not full, right courses
count = 0
#print(len(open_right_courses))
for row in open_right_courses:
info_list = row.find_elements_by_tag_name("td")
try:
if int(info_list[14].text) > 0:
open_right_not_full_courses.append(row)
count += 1
except ValueError:
continue
if count == 0:
return CourseStatus.FULL
# check if the course is available
return CourseStatus.AVAILABLE
# close after done checking
driver.quit()
def main(url, base_url):
data = []
s_url = []
gid_lists = getdata(url, base_url)
#合成手机端新闻内容链接
# print(len(gid_lists))
for j in range(len(gid_lists)):
source_url = base_url + gid_lists[j] + '/'
s_url.append(source_url)
#selenuim配置
option = ChromeOptions()
prefs = {
"profile.managed_default_content_settings.images": 2,
'permissions.default.stylesheet': 2
}
option.add_argument(
'user-agent="Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Mobile Safari/537.36"'
)
option.add_experimental_option("prefs", prefs)
option.add_experimental_option('excludeSwitches', ['enable-automation'])
header = random.choice(header_list)
option.add_argument(header)
option.add_argument('--headless')
option.add_argument('--disable-gpu')
driver = Chrome(options=option)
j = 0
last_content = ""
i = 0
repeat_num = 0
#循环遍历链接获取文章内容及时间
#redis配置
rdb = redis.Redis(host='localhost', port=6379, decode_responses=True)
while i < len(s_url):
# print('正在下载第', i + 1, '个题目:')
# print(s_url[i])
#去重
if rdb.exists(s_url[i]):
# print('数据重复， code: 0')
i = i + 1
continue
else:
driver.get(s_url[i])
# 隐性等待，最长等5秒
driver.implicitly_wait(5)
tree = etree.HTML(driver.page_source)
page_list = ''
#文章p列表
page_list = tree.xpath('//article//p//text()')
#新闻时间列表
ttime = tree.xpath(
'//div[@class="article-sub"]/span[last()]/text()')
#过滤视频内容
if ttime:
# print("时间",ttime)
content = ''
for p in page_list:
content = content + p
# print(content)
# 防止重复，重复次数最大不能超过5
if last_content == content:
# print("重复")
repeat_num = repeat_num + 1
if repeat_num > 5:
i = i + 1
repeat_num = 0
continue
if last_content != content:
repeat_num = 0
# 判断文章内容是否为空，如果为空则不加入数据库
if content.strip() == '':
i = i + 1
continue
new = {
'hotspot_data': {
'source': '今日头条',
'docid': gid_lists[i],
'url': s_url[i],
'time': ttime[0]
},
'content': content
}
data.append(new)
# print('下载成功', content)
data[j]['content'] = content
rdb.set(s_url[i], 1)
rdb.expire(s_url[i], 1209600) #设置redis过期时间为14天
last_content = content
j = j + 1
i = i + 1
# print(data)
driver.quit()
# print("finish")
saveData(data)
def __init__(self) -> None:
logging.debug("driver.create/start")
options = ChromeOptions()
binary_location = os.environ.get("CHROME_BINARY_LOCATION", None)
if not (binary_location is None):
options.binary_location = binary_location
# 必須
options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("--disable-gpu")
options.add_argument("--single-process")
# options.add_argument("--disable-setuid-sandbox")
# options.add_argument("--disable-dev-shm-usage")
# エラーの許容
options.add_argument("--ignore-certificate-errors")
options.add_argument("--allow-running-insecure-content")
options.add_argument("--disable-web-security")
# headlessでは不要そうな機能
options.add_argument("--disable-desktop-notifications")
options.add_argument("--disable-extensions")
# UA
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" # noqa
options.add_argument("--user-agent=" + user_agent)
# 言語
options.add_argument("--lang=ja")
# 画像を読み込まないで軽くする
options.add_argument("--blink-settings=imagesEnabled=false")
# chromedriver生成
executable_path = os.environ.get("CHROME_DRIVER_LOCATION", None)
self.driver = Chrome(options=options, executable_path=executable_path)
logging.debug("driver.create/end")
def get_driver_instance(self):
if self.driver is None:
if self.group_data.GLOBAL['Browser']['Name'] == 'firefox':
firefox_options = FirefoxOptions()
firefox_options.add_argument(argument='--width=1920')
firefox_options.add_argument(argument='--height=1080')
if self.group_data.GLOBAL['Browser']['headless'] is True:
firefox_options.headless = True
firefox_options.add_argument("--window-size=1920,1080")
if self.group_data.GLOBAL['Browser']['Remote'] is True:
self.driver = webdriver.Remote(
command_executor='http://127.0.0.1:4444/wd/hub',
options=firefox_options,
desired_capabilities=DesiredCapabilities.FIREFOX,
browser_profile=None,
proxy=None,
keep_alive=False,
file_detector=None)
else:
self.driver = webdriver.Firefox(
options=firefox_options,
desired_capabilities=DesiredCapabilities.FIREFOX)
self.driver = webdriver.Remote(
command_executor='http://127.0.0.1:4444/wd/hub',
desired_capabilities=DesiredCapabilities.FIREFOX,
options=firefox_options,
)
if self.group_data.GLOBAL['Browser']['Name'] == 'chrome':
# Chrome browser options
options = ChromeOptions()
options.set_capability("acceptInsecureCerts", True)
if self.group_data.GLOBAL['Browser']['headless'] is True:
options.headless = True
options.add_argument(argument='--width=1920')
options.add_argument(argument='--height=1080')
options.add_argument("--window-size=1920,1080")
# chrome_options.add_argument('--ignore-certificate-errors')
else:
options.headless = False
capabilities = DesiredCapabilities.CHROME
capabilities['goog:loggingPrefs'] = {
'browser': 'ALL',
'performance': 'ALL',
'server': 'ALL',
'client': 'ALL'
}
if self.group_data.GLOBAL['Browser']['Remote'] is True:
self.driver = webdriver.Remote(
command_executor='http://127.0.0.1:4444/wd/hub',
proxy=None,
options=options,
desired_capabilities=capabilities,
)
else:
self.driver = webdriver.Chrome(
options=options, desired_capabilities=capabilities)
if self.group_data.GLOBAL['Browser']['headless'] is False:
self.driver.maximize_window()
return self.driver
def _login(self):
l = self.l
option = ChromeOptions()
option.add_argument('-headless')
option.add_argument('--no-sandbox')
option.add_experimental_option('excludeSwitches', ['enable-automation'])
driver = Chrome(options=option, executable_path='/home/hr58/chromedriver')
self.driver = driver
driver.get('https://passport.58.com/login')
driver.find_elements_by_xpath('/html/body/div[1]/div[1]/div[2]/div/img')[0].click()
driver.find_elements_by_xpath('//*[@id="phonenum"]')[0].click()
ele = driver.find_elements_by_xpath('//*[@id="phone"]')[0]
phonenum = self.yima.generate_phone()
# print('phone:', phonenum)
l.info(f"get phone: {phonenum}")
for character in phonenum:
ele.send_keys(character)
time.sleep(0.1)
driver.save_screenshot('./c_phone.png')
# 点击发送验证码
# driver.find_elements_by_xpath('/html/body/div[1]/div[1]/div[3]/div[3]/div[1]/span')[0].click()
driver.find_element_by_class_name('getcode').click()
time.sleep(1.1)
driver.save_screenshot('./send_sms.png')
code = self.yima.get_message(phone=phonenum)
l.info(f"get code: {code}")
if not code:
l.error(f"get code timeout.")
return False
ele = driver.find_elements_by_xpath('//*[@id="mobilecode"]')[0]
for character in code:
ele.send_keys(character)
time.sleep(0.1)
time.sleep(1.1)
driver.find_elements_by_xpath('//*[@id="btn_phonenum"]')[0].click()
l.info('has clicked login button.')
time.sleep(5)
driver.get('https://employer.58.com/resumesearch?PGTID=0d000000-0000-02bf-9f94-8c7003dc986f&ClickID=29')
time.sleep(10)
driver.save_screenshot('./login.png')
self.yima.release_num(phonenum)
if 'employer.58.com/resumesearch' not in driver.current_url:
l.error(f"login failed, current url: {driver.current_url}")
return False
# driver.find_elements_by_xpath('/html/body/div[6]/div[1]/div[2]')[0].click()
# time.sleep(5)
# driver.find_elements_by_xpath('/html/body/div[2]/div[2]/div/div[3]/div[2]/ul/li[1]/div[1]/div[3]/p[1]/span[1]')[
# 0].click()
cookie_str = driver.execute_script('return document.cookie')
# cookies = {i['name']: i['value'] for i in driver.get_cookies()}
self.l.info(f"get cookies: {cookie_str}")
driver.quit()
# return cookies
return cookie_str
# -*- coding:utf-8 -*-
"""
/home/dfs/appilcations/python/virtualenvs/venvdl/bin/python3
workon venvdl
deactivate
rmvirtualenv venvdl
wget https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/libnvinfer7_7.1.3-1+cuda11.0_amd64.deb
"""
from selenium import webdriver
from selenium.webdriver import ChromeOptions
option = ChromeOptions()
option.add_argument("--disable-blink-features=AutomationControlled")
option.add_experimental_option('excludeSwitches', ['enable-automation'])
option.add_experimental_option('useAutomationExtension', False)
# option.add_argument('--headless')
browser = webdriver.Chrome(options=option)
browser.execute_cdp_cmd(
'Page.addScriptToEvaluateOnNewDocument', {
'source':
'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'
})
headers = {
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'en-US,en;q=0.8',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
from inputjadwal import inputJadwalUjian
from selenium.webdriver import ChromeOptions, Chrome
from emaildosen import *
from makefile import *
###########################
###### Buat Selenium ######
###########################
opts = ChromeOptions()
opts.add_argument("--headless")
opts.add_experimental_option("detach", True)
driver = Chrome(options=opts)
############################
###### Buat Per Prodi ######
############################
filters = {'tahun': '20192', 'jenis': '1', 'program': 'REG'}
prodis = [
'D4 Teknik Informatika', 'D4 Manajemen Perusahaan', 'D3 Logistik Bisnis',
'D4 Logistik Bisnis'
]
# inputJadwalUjian(driver, filters, "jadwal_uts_coba.xls")
# Buat generate PDF
# makeFile(driver, prodis, filters)
# Buat ngirim pdf ke email
# sendFileUjian(prodis, filters)
############################
def driver(self):
options = ChromeOptions()
options.add_argument('--headless')
wd = webdriver.Chrome(options=options)
return wd
# FIXME
UNRELEVANT_FILENAME = join(LOCATION_DIR, "unrelevant.txt".format(CITY))
# FIXME
LOCATIONS_PATH = join(LOCATION_DIR, "{}.txt")
N_PROBE = 10
TIMEOUT = 30
FLOAT_RE = "(\d+\.\d+)"
if not os.path.exists(LOCATION_DIR):
os.makedirs(LOCATION_DIR)
options = ChromeOptions()
options.add_argument("headless")
driver = Chrome(chrome_options=options)
driver.set_page_load_timeout(TIMEOUT)
driver.implicitly_wait(TIMEOUT)
driver.get(COUNTRY_EXPLORE_PATH)
print("Expanding areas list...")
while True:
try:
driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);") #
driver.find_element_by_css_selector(
"a[href*='{}?page=']".format(COUNTRY_EXPLORE_LINK)).click() #
time.sleep(1)
except:
from os import path
# Pasted from stores/amazon.py
from chromedriver_py import binary_path # this will get you the path variable
from selenium import webdriver
from selenium.webdriver import ChromeOptions
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
options = Options()
options.page_load_strategy = "eager"
chrome_options = ChromeOptions()
chrome_options.add_argument("--disable-application-cache")
chrome_options.add_experimental_option("excludeSwitches",
["enable-automation"])
chrome_options.add_experimental_option("useAutomationExtension", False)
chrome_options.add_experimental_option("detach", True)
prefs = {"profile.managed_default_content_settings.images": 2}
chrome_options.add_experimental_option("prefs", prefs)
from utils.logger import log
AUTOBUY_CONFIG_PATH = "autobuy_config.json"
AUTOBUY_CONFIG_KEYS = [
"NVIDIA_LOGIN",
"NVIDIA_PASSWORD",
"FULL_AUTOBUY",
"CREDITCARD_NUMBER",
self.driver.get(challenge)
challenge_title, challenge_description, challenge_solution = self._get_challenge_info()
self._save_info(tutorial_name, challenge_title, challenge_description, challenge_solution, extension)
except Exception as e:
print(f"The Challenge {i+1} Couldn't be retrevied")
print(e)
sleep(3)
### change this one to your need
# if your driver is already in your path
# you won't need to worry with the path_to_driver
# variable, therefore just leave it as a empty string
challenge_list_url = 'https://www.hackerrank.com/domains/tutorials/30-days-of-code?filters%5Bstatus%5D%5B%5D=solved&filters%5Bsubdomains%5D%5B%5D=30-days-of-code&badge_type=30-days-of-code'
tutorial_name = '30-days-of-code-cpp'
extension = 'cpp'
path_to_driver = 'chromedriver_linux64/chromedriver'
options = ChromeOptions()
options.add_argument('--start-maximized')
sc = scrapper(path_to_driver, options=options)
sc.login(username, password)
sleep(5)
sc.run(challenge_list_url, tutorial_name, extension)
sc.driver.close()
def get_driver(username, password):
options = ChromeOptions()
options.binary_location = GOOGLE_CHROME_PATH
options.add_argument('--headless')
options.add_argument("start-maximized") # open Browser in maximized mode
options.add_argument("disable-infobars") # disabling infobars
options.add_argument("--disable-extensions") # disabling extensions
options.add_argument("--no-sandbox")
options.add_argument(
"--disable-dev-shm-usage") # overcome limited resource problem
driver = Chrome(executable_path=DRIVER_PATH, chrome_options=options)
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.implicitly_wait(10)
# Login
driver.get(LOGIN_URL)
usernameField = driver.find_element_by_xpath(USERNAME_PATH)
usernameField.send_keys(username)
passwordField = driver.find_element_by_xpath(PASSWORD_PATH)
passwordField.send_keys(password)
passwordField.send_keys(Keys.RETURN)
print("SUCCESS: login by", username)
return driver
from threading import Thread
import time
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common import desired_capabilities
import unittest
desired_capabilities = webdriver.DesiredCapabilities.CHROME
desired_capabilities['version']='latest'
desired_capabilities['platform']='WINDOWS'
desired_capabilities['name']= 'Test Selenium'
Options = ChromeOptions()
Options.add_argument("start-maximized")
Options.add_argument("disable-infobars")
xpath_App_Sec = '//*[@id="footer-sitemap-policy-content-content"]/div/ul/li[1]/a'
# driver = webdriver.Chrome(chrome_options=Options,executable_path="WebDriver/chromedriver_235")
driver = webdriver.Remote(desired_capabilities=desired_capabilities,command_executor='http://localhost:4444/wd/hub')
driver.get('https://google.com')
time.sleep(5)
element = driver.find_element_by_link_text('About')
element.click()
driver.maximize_window()
AppSec = driver.find_element_by_xpath(xpath_App_Sec)
# element.location_once_scrolled_into_view
# ActionChains(driver).key_down(Keys.CONTROL).click(element).key_up(Keys.CONTROL).perform()
# ActionChains(driver).double_click(element).move_by_offset(20,30).perform()
from selenium.common.exceptions import TimeoutException
chrome_driver_path = os.environ["CHROME_DRIVER_PATH"]
chrome_binary_location = os.environ["CHROME_BINARY_LOCATION"]
line_url = "chrome-extension://ophjlpahpchlmihnnnihgmmeilfjmjjc/index.html"
# confidential ##################
login_e_mail = "*****@*****.**"
login_pwd = "dummypass"
#################################
options = ChromeOptions()
options.add_extension("chrome_extensions/LINE-Chrome_2.4.3.0.crx")
# options.add_argument("--headless") # headlessモードを使用する
options.add_argument("--disable-gpu") # headlessモードで暫定的に必要なフラグ(そのうち不要になる)
options.add_argument("--start-maximized") # 起動時にウィンドウを最大化する
options.add_argument("--disable-dev-shm-usage")
# options.add_argument("--no-sandbox")
# options.add_argument("--blink-settings=imagesEnabled=false")
options.binary_location = chrome_binary_location
# ユーザプロファイルのフォルダ名(実行フォルダに作成されます)
user_profile = 'UserProfile'
# Optionでユーザプロファイルの場所を指定する
options.add_argument('--user-data-dir=' + user_profile)
try:
chrome_driver = Chrome(executable_path=chrome_driver_path, options=options)
chrome_driver.implicitly_wait(15)
class ProductsChrome(LiveServerTestCase):
"""functional tests using selenium and TestCase."""
def setUp(self):
"""Method called to prepare the test fixture."""
self.options = ChromeOptions()
# self.options.add_experimental_option("excludeSwitches",
# ["enable-logging"])
self.options.add_argument('--headless')
self.options.add_argument('--disable-gpu')
self.options.add_argument('--remote-debugging-port=9222')
self.options.add_argument('--window-size=1920x1080')
self.browser = Chrome(chrome_options=self.options)
User = get_user_model()
self.user = User.objects.create_user(username='******',
email='*****@*****.**',
password='******')
self.category = Category(id_category="1à",
name="category_test",
products=1,
visible=True)
self.category.save()
for id_product in range(7):
self.product = Product(id_product="key%s" % id_product,
product_name_fr="test_%s" % id_product,
nutriscore_score=0,
nutriscore_grade='A',
brands='brand_test')
self.product.save()
self.product.categories.add(self.category)
def tearDown(self):
"""Method called immediately after the test method has been called and
the result recorded."""
self.browser.quit()
def login_user(self, email, pwd):
"""method simulating the connection of a user."""
time.sleep(2)
self.browser.get("%s%s" %
(str(self.live_server_url), '/accounts/login/'))
username_input = self.browser.find_element_by_id('id_username')
password_input = self.browser.find_element_by_id('id_password')
submission_button = self.browser.find_element_by_class_name(
'btn-success')
username_input.send_keys(email)
password_input.send_keys(pwd)
submission_button.click()
def search_product(self, action_id):
""" method simulating the search for a product
param: action_id (search-form or search-nav) """
time.sleep(2)
self.browser.get(str(self.live_server_url))
# search
search_input = self.browser.find_element_by_id(action_id)
search_input.send_keys('test_')
if action_id == 'search-form':
submission_button = self.browser.find_element_by_class_name(
'btn-primary')
submission_button.click()
else:
search_input.send_keys(Keys.ENTER)
time.sleep(2)
def test_search_product_with_form(self):
"""test search from form."""
self.search_product('search-form')
html = self.browser.page_source
self.assertInHTML(
"""
<h4 class="text-white">Votre recherche a renvoyé <strong>7</strong>
produits correspondant à <strong>test_ </strong></h4>""", html)
def test_search_product_with_nav(self):
"""test the search from the nav bar."""
self.search_product('search-nav')
html = self.browser.page_source
self.assertInHTML(
"""
<h4 class="text-white">Votre recherche a renvoyé <strong>7</strong>
produits correspondant à <strong>test_ </strong></h4>""", html)
def test_search_product_and_page_next(self):
"""test next page with search."""
time.sleep(2)
self.browser.get(
"%s%s" %
(str(self.live_server_url), '/products/search/?search=test'))
html = self.browser.page_source
# next page
next_button = self.browser.find_element_by_class_name('page-link')
next_button.click()
time.sleep(2)
html = self.browser.page_source
current_url = self.browser.current_url
expected_url = "%s%s" % (str(
self.live_server_url), '/products/search/?search=test&page=2')
self.assertEqual(expected_url, current_url)
self.assertInHTML(
"""
<p class="text-white bg-primary p-2">
Page 2 of 2.
</p>
""", html)
def test_get_substitutes(self):
"""test substitutes page."""
time.sleep(2)
self.browser.get(
"%s%s" %
(str(self.live_server_url), '/products/search/?search=test'))
a_button = self.browser.find_element_by_id('link-key1')
a_button.click()
time.sleep(1)
html = self.browser.page_source
current_url = self.browser.current_url
expected_url = "%s%s" % (str(
self.live_server_url), '/products/substitutes/key1')
self.assertEqual(expected_url, current_url)
self.assertInHTML(
"""
<h4 class="text-white">Vous pouvez remplacer <strong>test_1
(brand_test)</strong></h4>
""", html)
def test_add_fav(self):
"""test adding favorites."""
self.login_user('*****@*****.**', '123test')
time.sleep(2)
self.browser.get("%s%s" %
(str(self.live_server_url), '/products/key1'))
time.sleep(1)
fav_button = self.browser.find_element_by_id('add-fav')
fav_button.click()
time.sleep(1)
html = self.browser.page_source
self.assertInHTML("""
Retirer
""", html)
def run(self):
options = ChromeOptions()
options.add_argument('--test-type')
self.driver = Chrome(chrome_options=options)
self.perform_steps()
self.driver.close()
def restart_driver(self):
if self.browser != None:
self.browser.quit()
options = ChromeOptions()
options.add_argument("window-size=1920,1080")
options.add_argument("disable-gpu")
options.add_argument("disable-extensions")
options.add_argument("start-maximized")
if not self.debug:
options.add_argument('log-level=3')
options.add_argument('headless')
self.browser = Chrome(chrome_options=options)
try:
driver.find_element_by_id(element)
return flag
except:
flag = False
return flag
color_map = {"未开始": "lightgray", "进行中": "#1B7837", "已完成": "yellow"}
# 创建chrome启动选项
chrome_options = ChromeOptions()
# 指定chrome启动类型为headless 并且禁用gpu
ua = get_ua()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_experimental_option('excludeSwitches',
['enable-automation'])
chrome_options.add_argument('user-agent=' + ua)
# 调用环境变量指定的chrome浏览器创建浏览器对象
# 步骤:
# 下载chromedriver安装包：https://sites.google.com/a/chromium.org/chromedriver/downloads
# unzip chromedriver_map.zip
# mv chromedriver /usr/local/bin
# cd /usr/local/bin && chmod a+x chromedirver
# /usr/local/bin/chromedriver
driver = Chrome(options=chrome_options,
executable_path="/usr/local/bin/chromedriver")
def _init_chrome(self, adblock: bool = False, incognito: bool = False) -> Chrome:
empty_dir(str(Path.cwd() / 'chrome-data'))
chr_opt = ChromeOptions()
chr_opt.headless = self.headless
chr_opt.add_argument("--no-sandbox")
chr_opt.add_argument('log-level=2')
chr_opt.add_argument('--disable-logging')
chr_opt.add_argument('--disable-remote-fonts')
chr_opt.add_argument("--user-data-dir=chrome-data")
chr_opt.add_argument("--remote-debugging-port=9222")
chr_opt.add_argument("--disable-infobars")
chr_opt.add_argument("--disable-dev-shm-usage")
chr_opt.add_argument("--ignore-certificate-errors")
chr_opt.add_argument("--disable-gpu")
if adblock and not self.headless: # headless + extensions = crash
chr_opt.add_extension(
r'C:\Users\Roel\PycharmProjects\scrapers\tripadvisor\driver\ublock.crx.crx')
if incognito:
chr_opt.add_argument('--incognito')
if 'posix' in os.name:
chr_opt.binary_location = '/home/vries274/scrapers/tripadvisor/chrome/chrome-linux/chrome'
chrome = Chrome(executable_path=self.CHR_PATH, options=chr_opt)
chrome.set_window_size(1920, 1080)
print('\n --- Browser started --- \n')
return chrome
from selenium.webdriver import Chrome, ChromeOptions
path = "../driver/chromedriver"
ops = ChromeOptions()
ops.add_argument('--ignore-certificate-errors')
ops.add_argument("--disable-notifications")
driver = Chrome(executable_path=path, options=ops)
driver.fullscreen_window()
# driver.implicitly_wait(10)
# driver.set_script_timeout(100)
class GrabIt(urllib.request.FancyURLopener):
version = ('Mozilla/6.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/53.0.2526.111 Safari/547.36')
def download_file(self, url, path):
try:
self.urlretrieve = GrabIt().retrieve
self.urlretrieve(url, path)
except Exception as e:
print(str(e))
grab1 = GrabIt()
options = ChromeOptions()
options.add_argument('headless')
options.add_argument('disable-gpu')
driver = Chrome(chrome_options=options)
driver.get(args.url)
html = driver.page_source
soup = BeautifulSoup(html)
for img in soup.findAll('a', href=re.compile('.(.jpg|.png|.gif)')):
img2 = img['href']
imgbase = os.path.basename(img2)
endpoint = os.path.join(os.path.dirname(__file__), imgbase)
if os.path.isfile(endpoint):
print('file exists - skipping')
grab1.download_file(img2, endpoint)
print(img2)
time.sleep(1)
def get_web_driver(email,
password,
headless=False,
mfa_method=None,
mfa_input_callback=None,
wait_for_sync=True,
wait_for_sync_timeout=5 * 60,
session_path=None,
imap_account=None,
imap_password=None,
imap_server=None,
imap_folder="INBOX"):
if headless and mfa_method is None:
warnings.warn(
"Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms"
)
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd() + os.path.sep + 'chromedriver'
if _platform in ['win32', 'win64']:
executable_path += '.exe'
zip_type = CHROME_ZIP_TYPES.get(_platform)
if not os.path.exists(executable_path):
zip_file_url = CHROME_DRIVER_BASE_URL % (CHROME_DRIVER_VERSION,
zip_type)
request = requests.get(zip_file_url)
if request.status_code != 200:
raise RuntimeError(
'Error finding chromedriver at %r, status = %d' %
(zip_file_url, request.status_code))
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
if session_path is not None:
chrome_options.add_argument("user-data-dir=%s" % session_path)
driver = Chrome(chrome_options=chrome_options,
executable_path="%s" % executable_path)
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
try:
element = driver.find_element_by_link_text("Sign in")
except NoSuchElementException:
# when user has cookies, a slightly different front page appears
driver.implicitly_wait(0) # seconds
element = driver.find_element_by_link_text("Sign in")
driver.implicitly_wait(20) # seconds
element.click()
time.sleep(1)
email_input = driver.find_element_by_id("ius-userid")
# It's possible that the user clicked "remember me" at some point, causing
# the email to already be present. If anything is in the input, clear it
# and use the provided email, just to be safe.
# email_input.setAttribute("value", "")
email_input.clear()
email_input.send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
# An implicitly_wait is also necessary here to avoid getting stuck on
# find_element_by_id while the page is still in transition.
driver.implicitly_wait(1)
time.sleep(1)
# bypass "Let's add your current mobile number" interstitial page
try:
skip_for_now = driver.find_element_by_id(
'ius-verified-user-update-btn-skip')
skip_for_now.click()
except (NoSuchElementException, StaleElementReferenceException,
ElementNotVisibleException):
pass
driver.implicitly_wait(1) # seconds
try:
driver.find_element_by_id('ius-mfa-options-form')
try:
mfa_method_option = driver.find_element_by_id(
'ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id(
"ius-mfa-options-submit-btn")
mfa_method_submit.click()
if mfa_method == 'email' and imap_account:
mfa_code = get_email_code(imap_account,
imap_password,
imap_server,
imap_folder=imap_folder)
else:
mfa_code = (mfa_input_callback or
input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id(
"ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id(
"ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except Exception: # if anything goes wrong for any reason, give up on MFA
mfa_method = None
warnings.warn("Giving up on handling MFA. Please complete "
"the MFA process manually in the browser.")
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
try:
# Status message might not be present straight away. Seems to be due
# to dynamic content (client side rendering).
status_message = WebDriverWait(driver, 30).until(
expected_conditions.visibility_of_element_located(
(By.CSS_SELECTOR, ".SummaryView .message")))
WebDriverWait(driver, wait_for_sync_timeout).until(
lambda x: "Account refresh complete" in status_message.
get_attribute('innerHTML'))
except (TimeoutException, StaleElementReferenceException):
warnings.warn("Mint sync apparently incomplete after timeout. "
"Data retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
def get_web_driver(email, password, headless=False, mfa_method=None,
mfa_input_callback=None, wait_for_sync=True,
session_path=None, imap_account=None, imap_password=None,
imap_server=None, imap_folder="INBOX"):
if headless and mfa_method is None:
warnings.warn("Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms")
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd() + os.path.sep + 'chromedriver'
if _platform in ['win32', 'win64']:
executable_path += '.exe'
zip_type = CHROME_ZIP_TYPES.get(_platform)
if not os.path.exists(executable_path):
zip_file_url = CHROME_DRIVER_BASE_URL % (CHROME_DRIVER_VERSION, zip_type)
request = requests.get(zip_file_url)
if request.status_code != 200:
raise RuntimeError('Error finding chromedriver at %r, status = %d' %
(zip_file_url, request.status_code))
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
if session_path is not None:
chrome_options.add_argument("user-data-dir=%s" % session_path)
driver = Chrome(chrome_options=chrome_options, executable_path="%s" % executable_path)
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
try:
element = driver.find_element_by_link_text("Log In")
except NoSuchElementException:
# when user has cookies, a slightly different front page appears
driver.implicitly_wait(0) # seconds
element = driver.find_element_by_link_text("LOG IN")
driver.implicitly_wait(20) # seconds
element.click()
time.sleep(1)
email_input = driver.find_element_by_id("ius-userid")
# It's possible that the user clicked "remember me" at some point, causing
# the email to already be present. If anything is in the input, clear it
# and use the provided email, just to be safe.
# email_input.setAttribute("value", "")
email_input.clear()
email_input.send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
# An implicitly_wait is also necessary here to avoid getting stuck on
# find_element_by_id while the page is still in transition.
driver.implicitly_wait(1)
time.sleep(1)
# bypass "Let's add your current mobile number" interstitial page
try:
skip_for_now = driver.find_element_by_id('ius-verified-user-update-btn-skip')
skip_for_now.click()
except (NoSuchElementException, StaleElementReferenceException, ElementNotVisibleException):
pass
driver.implicitly_wait(1) # seconds
try:
driver.find_element_by_id('ius-mfa-options-form')
try:
mfa_method_option = driver.find_element_by_id('ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id("ius-mfa-options-submit-btn")
mfa_method_submit.click()
if mfa_method == 'email' and imap_account:
mfa_code = get_email_code(imap_account, imap_password, imap_server, imap_folder=imap_folder)
else:
mfa_code = (mfa_input_callback or input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id("ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id("ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except Exception: # if anything goes wrong for any reason, give up on MFA
mfa_method = None
warnings.warn("Giving up on handling MFA. Please complete "
"the MFA process manually in the browser.")
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
try:
# Status message might not be present straight away. Seems to be due
# to dynamic content (client side rendering).
status_message = WebDriverWait(driver, 30).until(
expected_conditions.visibility_of_element_located(
(By.CSS_SELECTOR, ".SummaryView .message")))
WebDriverWait(driver, 5 * 60).until(
lambda x: "Account refresh complete" in status_message.get_attribute('innerHTML')
)
except (TimeoutException, StaleElementReferenceException):
warnings.warn("Mint sync apparently incomplete after 5 minutes. Data "
"retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
def gen_default_chrome_options():
chrome_options = ChromeOptions()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--window-size=1280x1696")
chrome_options.add_argument("--disable-application-cache")
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--hide-scrollbars")
chrome_options.add_argument("--enable-logging")
chrome_options.add_argument("--log-level=0")
chrome_options.add_argument("--single-process")
chrome_options.add_argument("--ignore-certificate-errors")
chrome_options.add_argument("--homedir=/tmp")
return chrome_options