def start_browser(request, browser, timeout):
"""
Setup and run browser
:param request: pytest request
:param browser: name of browser that will start
:param timeout: page load timeout
:return wd: web driver
:return proxy: browsermob proxy
"""
chromedriver_path = '/home/zhukov/PycharmProjects/Otus/drivers/chromedriver'
firefoxdriver_path = '/home/zhukov/PycharmProjects/Otus/drivers/geckodriver'
url = urlparse(proxy.proxy).path
if browser == "chrome":
des_cap = DesiredCapabilities.CHROME
des_cap['loggingPrefs'] = {'performance': 'ALL'}
chrome_options().add_argument(argument='--proxy-server={}'.format(url))
options = chrome_options()
options.headless = True
wd = EventFiringWebDriver(
webdriver.Chrome(options=options,
executable_path=chromedriver_path,
desired_capabilities=des_cap), OpencartListener())
# command_executor = 'http://*****:*****@hub.browserstack.com:80/wd/hub'
# wd = EventFiringWebDriver(webdriver.Remote(command_executor,
# desired_capabilities={"browserName": "chrome",
# 'os': 'Linux', 'os_version': '16.04'}),
# OpencartListener())
wd.implicitly_wait(int(timeout))
else:
des_cap = DesiredCapabilities.FIREFOX
des_cap['loggingPrefs'] = {'performance': 'ALL'}
firefox_options().add_argument(
argument='--proxy-server={}'.format(url))
options = firefox_options()
options.headless = True
wd = EventFiringWebDriver(
webdriver.Firefox(options=options,
executable_path=firefoxdriver_path,
desired_capabilities=des_cap),
OpencartListener())
# command_executor = 'http://*****:*****@hub.browserstack.com:80/wd/hub'
# wd = EventFiringWebDriver(webdriver.Remote(command_executor,
# desired_capabilities={"browserName": "firefox",
# 'os': 'Windows', 'os_version': '10'}),
# OpencartListener())
wd.implicitly_wait(int(timeout))
wd.maximize_window()
request.addfinalizer(wd.quit)
return wd, proxy
def get_firefox_driver():
profile = FirefoxProfile()
profile.set_preference('browser.cache.disk.enable', False)
profile.set_preference('browser.cache.memory.enable', False)
profile.set_preference('browser.cache.offline.enable', False)
profile.set_preference('network.cookie.cookieBehavior', 2)
if os.environ.get("DEV"):
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.socks", "127.0.0.1")
profile.set_preference("network.proxy.socks_port", 8050)
profile.set_preference("network.proxy.socks_version", 5)
profile.update_preferences()
caps = DesiredCapabilities().FIREFOX
caps["pageLoadStrategy"] = "none"
options = firefox_options()
options.headless = True
options.add_argument('start-maximized')
path = os.path.dirname(__file__)
if platform.system() == "Linux":
path = os.path.join(path, r"driver/geckodriver")
else:
path = os.path.join(path, r"driver/geckodriver_mac")
driver = webdriver.Firefox(executable_path=path,
firefox_options=options,
desired_capabilities=caps,
firefox_profile=profile)
driver.set_window_size(1920, 1080)
driver.implicitly_wait(1)
return driver
def check_for_url(self):
if self.url != "":
if self.browser_type.get() == 0:
ch_options = chrome_options()
if self.headless.get() == 1:
ch_options.add_argument("--headless")
try:
self.driver = webdriver.Chrome("chrome1/chromedriver.exe", options=ch_options)
except:
try:
self.driver = webdriver.Chrome("chrome2/chromedriver.exe", options=ch_options)
except:
try:
self.driver = webdriver.Chrome("chrome3/chromedriver.exe", options=ch_options)
except:
print("NO WORKING CHROME DRIVER")
exit()
else:
ff_options = firefox_options()
if self.headless.get() == 1:
ff_options.add_argument("--headless")
try:
self.driver = webdriver.Firefox("firefox1/geckodriver.exe", options=ff_options)
except:
try:
self.driver = webdriver.Firefox("firefox2/geckodriver.exe", options=ff_options)
except:
try:
self.driver = webdriver.Firefox("firefox3/geckodriver.exe", options=ff_options)
except:
print("NO WORKING FIREFOX DRIVER")
exit()
self.driver.get(self.url)
self.main_loop()
def choose_webdriver(browser):
"""Choosing availabe webdriver in the webdriver_manager module.
Args:
browser (str): browser's name. One of ['chrome', 'chromium', 'firefox', 'ie'].
"""
if browser == 'chrome':
options = chrome_options()
# silent mode
options.add_argument('--headless')
# no printing log in the terminal
options.add_argument("--log-level=3")
driver = webdriver.Chrome(
executable_path=ChromeDriverManager().install(),
chrome_options=options)
elif browser == 'firefox':
options = firefox_options()
# silent mode
options.headless = True
driver = webdriver.Firefox(
executable_path=GeckoDriverManager().install(),
service_log_path=os.devnull,
options=options)
else:
raise Exception('Your browser is not supported.')
return (driver)
def start_browser():
if config.browser == 'firefox':
options = firefox_options()
options.headless = config.headless
if config.system == "linux":
browser = webdriver.Firefox(
options=options, executable_path=r"./include/geckodriver")
elif config.system == "windows":
browser = webdriver.Firefox(
options=options, executable_path=r".\include\geckodriver.exe")
elif config.browser == 'chrome':
options = chrome_options()
if config.headless:
options.add_argument('--headless')
if config.system == "linux":
browser = webdriver.Chrome(
options=options, executable_path=r"./include/chromedriver")
elif config.system == "windows":
browser = webdriver.Chrome(
options=options, executable_path=r".\include\chromedriver.exe")
else:
print("Error : wrong browser configured")
exit(1)
atexit.register(quit_browser, browser)
return browser
def start_selenium(driver_type='firefox',
seleniumwire_driver=False,
timeout=60,
is_headless=False,
proxy=None,
**kwargs):
if platform.system() == 'Linux':
display = Display(visible=0, size=(1280, 1024))
display.start()
if seleniumwire_driver:
driver_module = xhr_webdriver
else:
driver_module = webdriver
if driver_type == 'firefox':
driver_class = driver_module.Firefox
driver_path = Path(__file__).parent / '_webdrivers' / 'gecko.exe'
webdriver_manager = GeckoDriverManager()
options = firefox_options()
caps = webdriver.DesiredCapabilities.FIREFOX
elif driver_type == 'chrome':
driver_class = driver_module.Chrome
driver_path = Path(__file__).parent / '_webdrivers' / 'chrome.exe'
webdriver_manager = ChromeDriverManager()
options = chrome_options()
caps = webdriver.DesiredCapabilities.CHROME
else:
raise NotImplementedError(
f"Запуск драйвера {driver_type} не реализован")
if proxy:
caps['proxy'] = {
"proxyType": "MANUAL",
"httpProxy": proxy,
"sslProxy": proxy
}
driver_path = driver_path.resolve()
if not driver_path.exists():
driver_path.parent.mkdir(exist_ok=True)
cache_path = webdriver_manager.install()
shutil.copy(cache_path, str(driver_path))
options.headless = is_headless
driver = driver_class(options=options,
executable_path=str(driver_path),
capabilities=caps)
if not is_headless:
driver.maximize_window()
driver.set_page_load_timeout(timeout)
return driver
def grab_soup(url_, browser="firefox"):
"""
This function enables a driver (using Firefox or Chrome), goes to the URL, and retrieves the data after the JS is loaded.
:param url_: url to go to to retrieve data
:param browser: browser to use, defaults to firefox (requires geckodriver.exe on path)
:return:
soup - the data of the page
driver - the browser (process) instance
"""
if browser == 'chrome':
chromeOptions = chrome_options()
chromeOptions.add_experimental_option(
"prefs", {
"download.default_directory":
r"C:\Users\J_Ragbeer\Downloads",
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"safebrowsing.enabled": True
})
chromeOptions.add_argument("--disable-gpu")
chromeOptions.add_argument("--headless")
chromeOptions.add_argument('--no-sandbox')
chromeOptions.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome(
"C:/Users/Julien/PycharmProjects/practice/chromedriver95",
options=chromeOptions)
else:
firefoxOptions = firefox_options()
firefoxOptions.set_preference("browser.download.folderList", 2)
firefoxOptions.set_preference(
"browser.download.manager.showWhenStarting", False)
firefoxOptions.set_preference(
"browser.download.dir",
path.replace('/', '\\') + 'data\\downloads\\')
firefoxOptions.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"application/octet-stream,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
)
driver = webdriver.Firefox(options=firefoxOptions)
driver.get(url_) # go to the URL
html = driver.page_source
time.sleep(
1) # sleep for 1 second to ensure all JS scripts are loaded
html = driver.execute_script(
"return document.body.outerHTML;") # execute javascript code
soup_ = bs.BeautifulSoup(
html, 'lxml') # read the data as html (using lxml driver)
return soup_, driver
def start(self):
if self.browser == 'Chrome':
browser_options = chrome_options()
browser_options.add_argument("--headless")
browser_options.add_argument('--no-sandbox')
self.__driver = webdriver.Chrome(options=browser_options)
elif self.browser == 'Firefox':
browser_options = firefox_options()
browser_options.add_argument("--headless")
browser_options.add_argument('--no-sandbox')
self.__driver = webdriver.Firefox(options=browser_options)
else:
raise Exception('Please set browser to browser=Firefox or Chrome.')
logger.info(f'Using {self.browser}')
time.sleep(1)
logger.debug('Webdriver Started')
def make_driver():
driver = None
if config.use_proxy and config.browser != 'chrome':
raise Exception(f'proxy only supported in chrome')
if config.browser == 'chrome':
_chrome_options = chrome_options()
_chrome_options.add_argument("--disable-extensions")
_chrome_options.add_argument("--disable-gpu")
_chrome_options.add_argument("--disable-notifications")
_chrome_options.add_argument("--disable-logging")
_chrome_options.add_experimental_option('excludeSwitches', ['enable-logging'])
_chrome_options.add_experimental_option('prefs', {
'prefs': {'intl.accept_languages': 'en,en-US'},
'profile.default_content_setting_values.notifications' : 2,
'profile.managed_default_content_settings.images': 2})
#_chrome_options.add_argument(f"user-data-dir={config.chrome_profile_path}")
if config.use_proxy:
host, port, protocol = config.proxy_list[0]
_chrome_options.add_argument(f"--proxy-server={protocol}://{host}:{port}")
helper.thread_safe_print(f'using proxy: {protocol}://{host}:{port}')
if config.headless:
_chrome_options.add_argument("--start-maximized")
_chrome_options.add_argument("--headless")
driver = webdriver.Chrome(executable_path=config.chrome_driver, options=_chrome_options)
elif config.browser == 'firefox':
_firefox_profile = FirefoxProfile(config.firefox_profile_path)
_firefox_options = firefox_options()
_firefox_options.headless = config.headless
driver = webdriver.Firefox(_firefox_profile, options=_firefox_options)
else:
raise Exception(f'unknown brower "{config.browser}"')
if not config.skip_login:
login(driver)
return driver
def get_driver(head=False):
if head:
# options = webdriver.FirefoxOptions().set_headless()
# driver = webdriver.Firefox(firefox_options=options)
options = firefox_options()
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument("--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36")
driver = webdriver.Firefox(options=options)
# driver = webdriver.Firefox()
else:
options = Options()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument("--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36")
driver = webdriver.Chrome(options=options)
return driver
def init():
open("./file/DEBUG.txt", "w", encoding="utf-8").close()
try:
option = chrome_options()
option.add_argument("--headless")
v_driver = webdriver.Chrome(options=option)
except:
print_debug("[v_driver] don't detect Chrome", "yellow")
try:
option = firefox_options()
option.add_argument("--headless")
v_driver = webdriver.Opera(options=option)
except:
print_debug("[v_driver] don't detect Opera", "yellow")
try:
option = opera_options()
option.add_argument("--headless")
v_driver = webdriver.Firefox(options=option)
except:
print_debug("[v_driver] CANT FIND COMPATIBLE DRIVER -> EXIT",
"red")
e = pop_up(
"/msg{-txt:aucun moteur\n de recherche detecté \n(chrome, opera ou firefox) -title:VoltaireTaMere -link:https://www.google.com/intl/fr_fr/chrome/ -lock:0 -size:180x90}"
)
e.start_root()
e.root.mainloop()
exit()
return
v_driver.get("https://sites.google.com/view/voltairetamere/init")
v_driver.implicitly_wait(1)
init_command = v_driver.find_element_by_class_name("yaqOZd").text
print("init_command:", init_command)
v_driver.close()
if init_command[init_command.index("version:") + 8:] != found_data(
"./file/version.txt", "version", "str"):
init_command = "/msg{-txt:VoltaireTaMere doit être\n mis à jour -title:VoltaireTaMere -link:https://sites.google.com/view/voltairetamere/accueil -lock:1 -size:180x70}"
if init_command != "version:" + found_data("./file/version.txt", "version",
"str"):
w = pop_up(init_command)
w.start_root()
w.root.mainloop()
if w.option.lock:
exit()
def get_insta_cookies(self):
# login_url = "https://www.instagram.com/accounts/login/"
login_url = "https://www.instagram.com/gabryxx7/?__a=1"
# login_url = "https://www.google.com"
options = firefox_options()
# options = chrome_options()
options.headless = True
options.add_argument("user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36")
self.log.d("get_insta_cookies",f"Options created")
driver = selenium.webdriver.Firefox(options=options)
# driver = selenium.webdriver.Firefox(options=options)
self.log.d("get_insta_cookies",f"WebDriver created")
wait = WebDriverWait(driver, 10)
self.log.d("get_insta_cookies",f"Opening Firefox and navigating to {login_url}")
try:
driver.get(login_url)
## Print request headers
# for request in driver.requests:
# self.log.d("get_insta_cookies",request.url) # <--------------- Request url
# self.log.d("get_insta_cookies",request.headers) # <----------- Request headers
# self.log.d("get_insta_cookies",request.response.headers) # <-- Response headers
self.log.d("get_insta_cookies",f"Waiting for redirection...")
time.sleep(5)
with open("login_page.html", "w") as f:
f.write(driver.page_source)
self.log.d("get_insta_cookies",f"Waiting for clickable username field...")
username_field = wait.until(element_to_be_clickable(By.CSS_SELECTOR, "input[name='username']"))
username_field.clear()
username_field.send_keys(login_data["username"])
self.log.d("get_insta_cookies",f"Waiting for clickable password field...")
password_field = wait.until(element_to_be_clickable(By.CSS_SELECTOR, "input[name='password']"))
password_field.clear()
password_field.send_keys(login_data["password"])
self.log.d("get_insta_cookies",f"Waiting for clickable submit button...")
wait.until(element_to_be_clickable((By.XPATH, "//button[@type='submit'"))).click()
with open("login_page.html", "w") as f:
f.write(driver.page_source)
pickle.dump( driver.get_cookies() , open("cookies.pkl","wb"))
except Exception as e:
self.log.d("get_insta_cookies",f"Exception in getting cookies: {e}")
traceback.print_exc()
driver.quit()
def start_browser(request, browser):
"""
Setup and run browser
:param request: pytest request
:param browser: name of browser that will start
"""
chromedriver_path = '/home/support/Py_projects/Otus/drivers/chromedriver'
firefoxdriver_path = '/home/support/Py_projects/Otus/drivers/geckodriver'
if browser == "chrome":
options = chrome_options()
options.headless = True
wd = webdriver.Chrome(options=options,
executable_path=chromedriver_path)
else:
options = firefox_options()
options.headless = True
wd = webdriver.Firefox(options=options,
executable_path=firefoxdriver_path)
wd.maximize_window()
request.addfinalizer(wd.quit)
return wd
from selenium import webdriver
from selenium.webdriver import Proxy, FirefoxProfile
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.options import Options as firefox_options
p = Proxy()
p.http_proxy = "1.1.1.1"
fp = FirefoxProfile()
fp.accept_untrusted_certs = True
fp.assume_untrusted_cert_issuer = False
# fp.set_proxy(proxy)
chrome_options = Options()
firefox_options = firefox_options()
firefox_options.headless = True
chrome_options.headless = False
# driver = webdriver.firefox(executable_path=os.getcwd() + os.path.sep + "geckodriver_mac")
driver = webdriver.Chrome(options=chrome_options, executable_path=os.getcwd() + os.path.sep + "chromedriver_2.45")
driver.implicitly_wait(30)
driver.maximize_window()
driver.get("https://www.google.com")
print(driver.title)
driver.find_element_by_name("q").send_keys("ramnath gokul", Keys.SHIFT)
driver.find_element_by_name("q").send_keys(Keys.ENTER)
# driver.find_element_by_name("btnK").click()
time.sleep(2)
driver.quit()
def __init__(self,
url,
username,
password,
twilio_account_sid="",
twilio_auth_token="",
gmail_username="",
gmail_password="",
arg=None):
self.url = url
self.username = username
self.password = password
self.twilio_account_sid = twilio_account_sid
self.twilio_auth_token = twilio_auth_token
self.gmail_username = gmail_username
self.gmail_password = gmail_password
"""
Empty failed_hostnames list to be later used to store hostnames that were not updated
because of some issues
"""
self.failed_hostnames = []
# Checking if firefox or chrome has been chosen in the settings and running selected web driver
if settings.get("pref_webdriver").lower() == "firefox":
try:
# Checking if headless mode has been selected before initialising firefox web driver
if arg:
print("[*] Launching Firefox with headless mode")
# Setting web driver to run in headless mode using firefox Options() and passing it as an argument when initialising firefox web driver
firefox_opts = firefox_options()
firefox_opts.add_argument("--headless")
self.driver = webdriver.Firefox(options=firefox_opts)
else:
print("[*] Launching Firefox")
self.driver = webdriver.Firefox()
except:
print(
"""[X] Something went wrong with initialising the firefox web driver,
make sure the executable web driver for firefox has been added to
PATH in your system and that the current firefox web driver supports
the current version of firefox in your system e.g. download an older
version of firefox web driver
""")
sys.exit()
elif settings.get("pref_webdriver").lower() == "chrome":
try:
# Checking if headless mode has been selected before initialising chrome web driver
if arg:
print("[*] Launching Chrome with headless mode")
# Setting web driver to run in headless mode using chrome Options() and passing it as an argument when initialising chrome web driver
chrome_opts = chrome_options()
chrome_opts.add_argument("--headless")
self.driver = webdriver.Chrome(options=chrome_opts)
else:
print("[*] Launching Chrome")
self.driver = webdriver.Chrome()
except:
print(
"""[X] Something went wrong with initialising the chrome web driver,
make sure the executable web driver for chrome has been added to
PATH in your system and that the current chrome web driver supports
the current version of chrome in your system e.g. download an older
version of chrome web driver
""")
sys.exit()
else:
print(
"[X] Given web browser name in settings is not recognised must be either firefox or chrome"
)
sys.exit()
def setUp(self):
options = firefox_options()
options.add_argument('-headless')
self.browser = webdriver.Firefox(options=options)