def _proxy(self):
logger.info("proxy: {}".format(self.proxy))
if self.proxy and isinstance(self.proxy,
dict) and self.browser == 'chrome':
proxy = Proxy()
for k, v in self.proxy.items():
proxy.__setattr__(k, v)
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
return capabilities
elif self.proxy and isinstance(self.proxy,
dict) and self.browser == 'firefox':
profile = FirefoxProfile()
profile.set_preference('network.proxy.type', 1)
for k, v in self.proxy.items():
scheme = k.split("_")
host_details = v.split(":")
if len(scheme) > 1 and len(host_details) > 1:
profile.set_preference("network.proxy.%s" % scheme[0],
host_details[0])
profile.set_preference("network.proxy.%s_port" % scheme[0],
int(host_details[1]))
profile.update_preferences()
# profile.accept_untrusted_certs = True
return profile
def _create_profile(self):
"""Create profile object.
"""
profile = FirefoxProfile()
for name, value in self.default_profile.items():
profile.set_preference(name, value)
return profile
def driver_factory(browser):
if browser == "chrome":
logger = logging.getLogger('chrome_fixture')
logger.setLevel(LOG_LEVEL)
options = ChromeOptions()
options.headless = True
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
logger.info("Подготовка среды для запуска тестов...")
caps = DesiredCapabilities.CHROME
caps['loggingPrefs'] = {'performance': 'ALL', 'browser': 'ALL'}
options.add_experimental_option('w3c', False)
driver = EventFiringWebDriver(
webdriver.Chrome(desired_capabilities=caps, options=options),
MyListener())
logger.debug(
"Браузер Chrome запущен со следующими desired_capabilities:{}".
format(driver.desired_capabilities))
elif browser == "firefox":
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
options = FirefoxOptions()
options.headless = True
driver = webdriver.Firefox(options=options, firefox_profile=profile)
else:
raise Exception("Driver not supported")
return driver
def driver_factory(browser, executor):
if browser == "chrome":
logger = logging.getLogger('chrome_fixture')
logger.setLevel(LOG_LEVEL)
options = ChromeOptions()
options.headless = True
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
logger.info("Подготовка среды для запуска тестов...")
options.add_experimental_option('w3c', False)
driver = EventFiringWebDriver(
webdriver.Remote(command_executor=f"http://{executor}:4444/wd/hub",
desired_capabilities={
"browserName": browser,
"platform": "WIN10",
"platformName": "WIN10"
},
options=options), MyListener())
logger.debug(
"Браузер Chrome запущен со следующими desired_capabilities:{}".
format(driver.desired_capabilities))
elif browser == "firefox":
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
options = FirefoxOptions()
options.headless = True
driver = webdriver.Firefox(options=options, firefox_profile=profile)
else:
raise Exception("Driver not supported")
return driver
def click_to_reveal_it(url, css_selector) -> webdriver.Firefox.page_source:
"""
Uses selenium to render request in a browser
locate a link, move mouse pointer to the link
and clicks it.
"""
BROWSER_ZOOM_SETTING = "layout.css.devPixelsPerPx"
ZOOM_FACTOR = "0.4"
TIME_SECS = 5
options = Options()
options.add_argument("-headless")
profile = FirefoxProfile()
profile.set_preference(BROWSER_ZOOM_SETTING, ZOOM_FACTOR)
browser = webdriver.Firefox(options=options, firefox_profile=profile)
browser.maximize_window()
browser.implicitly_wait(TIME_SECS)
browser.get(url)
try:
# TODO: add wait implementation (research: staleness_of)
time.sleep(1)
link = browser.find_element_by_css_selector(css_selector)
webdriver.ActionChains(browser).move_to_element(link).click(link).perform()
time.sleep(1)
return browser.page_source
except NoSuchElementException:
return browser.page_source
finally:
browser.quit()
def load_localstorage(self, session_id):
sessions = json.load(open(self.sessions_file))
storage_path = sessions[str(session_id)]["session_path"]
url = sessions[str(session_id)]["web_url"]
# Setting useragent to the same one the session saved with
useragent = sessions[str(session_id)]["useragent"]
profile = FirefoxProfile()
profile.set_preference("general.useragent.override", useragent)
localStorage = pickle.load(open(storage_path, "rb"))
try:
browser = Firefox(profile)
except:
error("Couldn't open browser to view session!")
return
browser.get(url)
browser.delete_all_cookies()
browser.execute_script(
"window.localStorage.clear()") # clear the current localStorage
for key, value in localStorage.items():
browser.execute_script(
"window.localStorage.setItem(arguments[0], arguments[1]);",
key, value)
status(f"Session {session_id} loaded")
browser.refresh()
self.browsers.append(browser)
def _setup_proxy(self, browser):
"""Setup proxy for Firefox or Chrome
:type browser: str
:param browser: Browser name
:rtype: selenium.webdriver.FirefoxProfile or selenium.webdriver.ChromeOptions
:returns: Proxy settings
"""
host = "127.0.0.1"
port = "9050"
if browser == "firefox":
firefox_profile = FirefoxProfile()
# Direct = 0, Manual = 1, PAC = 2, AUTODETECT = 4, SYSTEM = 5
firefox_profile.set_preference("network.proxy.type", 1)
firefox_profile.set_preference("network.proxy.socks", host)
firefox_profile.set_preference("network.proxy.socks_port", int(port))
firefox_profile.update_preferences()
return firefox_profile
elif browser == "chrome":
proxy = f"socks5://{host}:{port}"
chrome_options = ChromeOptions()
chrome_options.add_argument(f"--proxy-server={proxy}")
return chrome_options
else:
logger.info(f"No proxy setting was done for {browser}")
def driver_factory(browser, executor_url, test_name):
if browser == "chrome":
logger = logging.getLogger('chrome_fixture')
logger.setLevel(LOG_LEVEL)
caps = {
"browserName": browser,
"version": "83.0",
"enableVnc": True,
"enableVideo": True,
"enableLog": True,
"screenResolution": "1280x720",
"name": test_name
}
driver = EventFiringWebDriver(
webdriver.Remote(command_executor=executor_url + "/wd/hub",
desired_capabilities=caps), MyListener())
logger.info(f"Start session {driver.session_id}")
elif browser == "firefox":
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
options = FirefoxOptions()
options.headless = True
driver = webdriver.Firefox(options=options, firefox_profile=profile)
else:
raise Exception("Driver not supported")
return driver
def __init__(self, browser_choice, card_to_search):
# Set up config file
try:
with open('config.yml', 'r') as config_file:
self.config = yaml.safe_load(config_file)
# Sanity check on card name
self.url = self.config[card_to_search]['url']
self.search_term = self.config[card_to_search]['search_term']
except KeyError:
print('{card} does not exist in the config file'.format(
card=card_to_search))
exit(1)
except IOError:
print('Config file is missing. Please download it from the source')
exit(1)
# Set up Selenium browser
if browser_choice == 'chrome':
chrome_options = ChromeOptions()
chrome_options.add_argument('--incognito')
self.browser = Chrome(chrome_options=chrome_options)
elif browser_choice == 'firefox':
firefox_profile = FirefoxProfile()
firefox_profile.set_preference('browser.privatebrowsing.autostart',
True)
self.browser = Firefox(firefox_profile=firefox_profile)
def prepare_browsers(headless: bool, driver_path: str,
twitter_profile_path: str) -> Browsers:
"""
Sets up browsers to search accounts
:param headless bool: Should search be performed in headless mode
:param driver_path: Path to geckodriver
:param twitter_profile_path: Path to twitter profile folder
:return: tuple of browsers, that are logged in LinkedIn and Xing
"""
logging.info("Running Twitter scraper from profile in %s",
twitter_profile_path)
driver_path = driver_path if driver_path else "geckodriver"
profile = FirefoxProfile()
twitter_profile = FirefoxProfile(twitter_profile_path)
twitter_profile.DEFAULT_PREFERENCES["frozen"][
"extensions.autoDisableScopes"] = 0
twitter_profile.set_preference("extensions.enabledScopes", 15)
logins = social_media_logins(driver_path, profile)
driver_options = FirefoxOptions()
driver_options.headless = headless
linked_in_driver = Firefox(options=driver_options,
firefox_profile=profile,
executable_path=driver_path)
xing_driver = Firefox(options=driver_options,
firefox_profile=profile,
executable_path=driver_path)
twitter_driver = Firefox(options=driver_options,
firefox_profile=twitter_profile,
executable_path=driver_path)
set_login_data(linked_in_driver, logins[0])
set_login_data(xing_driver, logins[1])
retoggleAllTheAddons(twitter_driver)
return Browsers(linked_in_driver, xing_driver, twitter_driver)
def create_driver():
#driver = webdriver.Chrome("chromedriver.exe")
custom_profile = FirefoxProfile()
custom_profile.set_preference("dom.disable_beforeunload", True)
driver = webdriver.Firefox(firefox_profile=custom_profile, executable_path="geckodriver.exe")
driver.implicitly_wait(10)
return driver
def test_profiles_do_not_share_preferences():
profile1 = FirefoxProfile()
profile1.accept_untrusted_certs = False
profile2 = FirefoxProfile()
# Default is true. Should remain so.
assert profile2.default_preferences[
"webdriver_accept_untrusted_certs"] is True
def start(cls, url, driver=""):
if cls.mode == "FireFox":
default_profile = {
'browser.usedOnWindows10': False,
'browser.usedOnWindows10.introURL': 'https://www.google.com/',
'startup.homepage_welcome_url.additional': 'about:blank',
'browser.startup.homepage_override.mstone': 'ignore',
}
profile = FirefoxProfile()
for name, value in default_profile.items():
profile.set_preference(name, value)
cls.driver = webdriver.Firefox(profile)
elif cls.mode == "Chrome":
try:
if driver != "":
chromedriver = driver
L.info(chromedriver)
os.environ["webdriver.chrome.driver"] = chromedriver
cls.driver = webdriver.Chrome(chromedriver)
else:
cls.driver = webdriver.Chrome()
except Exception as e:
L.warning(str(e))
raise SeleniumError(
"Can't find Selenium Driver Mode. %s" % cls.mode)
else:
raise SeleniumError(
"Can't find Selenium Driver Mode. %s" % cls.mode)
cls.driver.implicitly_wait(DEFAULT_WAIT)
cls.driver.set_window_size(WINDOW_SIZE_WIDTH, WINDOW_SIZE_HEIGHT)
cls.driver.get(url)
time.sleep(5)
def launch_application(browser_name, app_url):
global driver
log.info("in init method of selenium base")
try:
if browser_name == "chrome":
option = ChromeOptions()
option.add_argument("start-maximized")
option.add_argument("--ignore-certificate-errors")
option.add_argument("--disable-extensions")
option.add_argument("--disable-infobars")
option.add_argument("disable-notifications")
driver = Chrome(executable_path="./drivers/chromedriver.exe",
options=option)
log.info("chrome browser is launch successfully")
elif browser_name == "firefox":
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
options = FirefoxOptions()
options.add_argument("start-maximized")
driver = Firefox(executable_path="./drivers/geckodriver.exe")
log.info("firefox browser is launch successfully")
elif browser_name == "ie":
driver = Ie(executable_path="./drivers/IEDriverServer.exe")
else:
log.error("browser name is incorrect", browser_name)
except WebDriverException:
log.critical("exception", WebDriverException)
driver.implicitly_wait(5)
driver.get(app_url)
def setOptions(self):
if self.browser == "Chrome":
# self.options.add_argument('--headless')
self.options.add_argument('--disable-gpu')
self.options.add_argument('--ignore-certificate-errors')
self.options.add_argument('--allow-running-insecure-content')
self.options.add_argument('--process-per-site')
self.options.add_argument('--disable-plugins')
self.options.add_argument('--hide-scrollbars')
self.options.add_argument('--log-level=3')
self.options.add_argument('--silent')
elif self.browser == "Firefox":
self.capabilities = DesiredCapabilities.FIREFOX.copy()
self.capabilities['platform'] = "WINDOWS"
print(self.capabilities)
self.options = Options()
profile = FirefoxProfile()
profile.set_preference(
'security.insecure_field_warning.contextual.enabled', False)
profile.set_preference('security.insecure_password.ui.enabled',
False)
profile.set_preference('dom.ipc.processCount', 1)
profile.set_preference('browser.tabs.remote.autostart.2', False)
self.options.add_argument("-headless")
self.options.add_argument("-new-tab")
# self.options.add_argument("-browser")
self.options.add_argument("-no-remote")
self.options.add_argument("-no-first-run")
self.options.add_argument("-app")
def create_options(self):
super().debug_begin()
options = FirefoxOptions()
options.add_argument("--start-maximized")
if super()._language is not None:
options.add_argument("--lang=" + super()._language)
if super()._headless:
options.add_argument("-headless")
options.add_argument("--disable-gpu")
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
for file_name in self.__plugin_files:
if '\\.' not in file_name:
file_name += '.xpi'
profile.add_extension(os.getcwd() + '/' + file_name)
options.profile = profile
if super()._use_proxy:
# options.add_argument('--ignore-certificate-errors')
pass
super().debug_end()
return options
def main(cli=True, observations=2):
"""
Main code, including selenium driver initialization.
By default, outputs lines to stdout. If cli=False, the function returns a list of tuples with the resulting lines.
"""
try:
# XXX: remove this line to keep the log and continue from where you left off.
os.remove(LOG_PATH)
except OSError:
pass
profile = FirefoxProfile()
# The DesiredCapabilities approach does not work with firefox, see https://github.com/mozilla/geckodriver/issues/284
profile.set_preference("devtools.console.stdout.content", True)
options = Options()
# options.headless = True
driver = webdriver.Firefox(firefox_profile=profile, options=options)
try:
lines = collect(driver, observations)
if not cli:
return lines
for line in lines:
print(line)
return None
finally:
driver.close()
def create(self) -> WebDriver:
pixel_density = current_app.config["WEBDRIVER_WINDOW"].get(
"pixel_density", 1)
if self._driver_type == "firefox":
driver_class = firefox.webdriver.WebDriver
options = firefox.options.Options()
profile = FirefoxProfile()
profile.set_preference("layout.css.devPixelsPerPx",
str(pixel_density))
kwargs: Dict[Any, Any] = dict(options=options,
firefox_profile=profile)
elif self._driver_type == "chrome":
driver_class = chrome.webdriver.WebDriver
options = chrome.options.Options()
options.add_argument(
f"--force-device-scale-factor={pixel_density}")
options.add_argument(
f"--window-size={self._window[0]},{self._window[1]}")
kwargs = dict(options=options)
else:
raise Exception(
f"Webdriver name ({self._driver_type}) not supported")
# Prepare args for the webdriver init
# Add additional configured options
for arg in current_app.config["WEBDRIVER_OPTION_ARGS"]:
options.add_argument(arg)
kwargs.update(current_app.config["WEBDRIVER_CONFIGURATION"])
logger.info("Init selenium driver")
return driver_class(**kwargs)
def test_autodetect_proxy_is_set_in_profile():
profile = FirefoxProfile()
proxy = Proxy()
proxy.auto_detect = True
profile.set_proxy(proxy)
assert profile.default_preferences["network.proxy.type"] == ProxyType.AUTODETECT['ff_value']
def test_pac_proxy_is_set_in_profile():
profile = FirefoxProfile()
proxy = Proxy()
proxy.proxy_autoconfig_url = 'http://some.url:12345/path'
profile.set_proxy(proxy)
assert profile.default_preferences["network.proxy.type"] == ProxyType.PAC['ff_value']
assert profile.default_preferences["network.proxy.autoconfig_url"] == 'http://some.url:12345/path'
def test_autodetect_proxy_is_set_in_profile():
profile = FirefoxProfile()
proxy = Proxy()
proxy.auto_detect = True
profile.set_proxy(proxy)
assert profile.default_preferences[
"network.proxy.type"] == ProxyType.AUTODETECT['ff_value']
def __init__(self):
self.profile = FirefoxProfile()
if enable_change_proxy:
self.profile = self.change_proxy(host, port, self.profile)
else:
self.profile = self.clear_proxy(self.profile)
self.profile.update_preferences()
self.driver = webdriver.Firefox(firefox_profile=self.profile)
self.driver.maximize_window()
def __init__(self):
self.geo_disabled = FirefoxProfile()
self.geo_disabled.set_preference("geo.enabled", True)
self.geo_disabled.set_preference("geo.provider.use_corelocation", True)
self.geo_disabled.set_preference("geo.prompt.testing", True)
self.geo_disabled.set_preference("geo.prompt.testing.allow", True)
self.driver = webdriver.Firefox(
executable_path='C:\\Users\\Allen\\OneDrive\\桌面\\geckodriver',
firefox_profile=self.geo_disabled)
def get_search_results(minresults=40):
"""Collect property urls and types by going through the search result pages of new houses and appartments,
stopping when having reached the minimum number of results and returning a dictionary of {'url1':True/False, 'url2':True/False, ...}.
True means house. False means apartment. Without argument only the first page is collected (~60 results)"""
search_results = {}
result_count = 0
# set on which page to start the search
page_number = 1
options = FirefoxOptions()
options.add_argument('-headless')
options.set_preference("dom.webdriver.enabled", False)
profile = FirefoxProfile('src/scraping/bolzyxyb.heroku')
profile.set_preference('useAutomationExtension', False)
driver = Firefox(firefox_binary='usr/lib/firefox/firefox',
options=options,
firefox_profile=profile)
driver.implicitly_wait(15)
# start the progress indicator and timeout logic
start_time = time.monotonic()
time_spent = 0
while result_count < minresults and time_spent < 1800:
# for each loop, scrape one results page of houses and one of appartments
# the results are added if they are not there yet
for houselink in get_page_urls(pagenr=page_number,
kind="house",
drv=driver):
if houselink not in search_results:
search_results[houselink] = True
for apartmentlink in get_page_urls(pagenr=page_number,
kind="apartment",
drv=driver):
if apartmentlink not in search_results:
search_results[apartmentlink] = False
result_count = len(search_results)
page_number += 1
# update progress indicator
time_spent = time.monotonic() - start_time
total_time_estimation = 1 / (result_count / minresults) * time_spent
if total_time_estimation > 1800:
capped_time = 1800
else:
capped_time = total_time_estimation
time_remaining = capped_time - time_spent
print(f"Finishing in {time_remaining/60:.1f} minutes")
driver.close()
print("Finished")
return search_results
def load_website(url, title):
config = read_config()
profile = FirefoxProfile()
proxy = Proxy()
proxy.proxy_autoconfig_url = config['proxy_url']
profile.set_proxy(proxy)
driver = Firefox(firefox_profile=profile)
driver.get(url)
assert title in driver.title
return driver
def test_pac_proxy_is_set_in_profile():
profile = FirefoxProfile()
proxy = Proxy()
proxy.proxy_autoconfig_url = 'http://some.url:12345/path'
profile.set_proxy(proxy)
assert profile.default_preferences["network.proxy.type"] == ProxyType.PAC[
'ff_value']
assert profile.default_preferences[
"network.proxy.autoconfig_url"] == 'http://some.url:12345/path'
def __call__(self, config, args):
profile = FirefoxProfile()
if config.user_agent:
profile.set_preference("general.useragent.override",
config.user_agent)
if config.ignore_ssl:
profile.accept_untrusted_certs = True
args["firefox_profile"] = profile
args["capabilities"] = args["desired_capabilities"]
args.pop("desired_capabilities")
return args
def __init__(self, cer_data, key_data):
self.temp_cer = self.temp_file(cer_data)
self.temp_key = self.temp_file(key_data)
profile = FirefoxProfile(settings.PROFILE_PATH)
for preference in settings.PROFILE_PREFERENCES:
profile.set_preference(*preference)
self._driver = Firefox(firefox_profile = profile)
self._driver.get(settings.GET_CFDI_URL)
def get_cookies(num):
# user = '******'
# pwd = 'lxq69688'
n = num % 2
users = ['[email protected]', '14785107068----lxq69688']
user = users[n]
one_user = (user).split('----')
user, pwd = one_user[0], one_user[1]
option = FirefoxProfile()
option.set_preference("dom.webnotifications.enabled", False)
browser = Firefox(option)
wait = WebDriverWait(browser, 120)
browser.delete_all_cookies()
browser.get(start_url)
submit = wait.until(
EC.presence_of_element_located(
(By.XPATH,
'//div[@class="W_login_form"]/div[@class="info_list login_btn"]/a'
)))
sleep(1)
submit.click()
user_element = wait.until(
EC.presence_of_element_located((By.XPATH, '//input[@id="loginname"]')))
# user_element.clear()
sleep(1)
user_element.send_keys(user)
print('账号:', user)
pwd_element = wait.until(
EC.presence_of_element_located(
(By.XPATH, '//input[@type="password"]')))
# pwd_element.clear()
sleep(1)
pwd_element.send_keys(pwd)
print('密码:', pwd)
submit = wait.until(
EC.presence_of_element_located(
(By.XPATH,
'//div[@class="W_login_form"]/div[@class="info_list login_btn"]/a'
)))
submit.click()
print('点击登陆！')
# sleep(2)
user_btn = wait.until(
EC.presence_of_element_located(
(By.XPATH, '//ul[@class="gn_nav_list"]/li[5]//em[2]')))
print('登陆用户：', user_btn.text)
#条件 确认是否登陆
cookies = browser.get_cookies()
cookie = ';'.join(i['name'] + '=' + i['value'] for i in cookies)
browser.close()
cookies = {
'Cookie': cookie,
}
return cookies
def __call__(self, config, args):
profile = FirefoxProfile()
if config.user_agent:
profile.set_preference("general.useragent.override",
config.user_agent)
if config.ignore_ssl:
profile.accept_untrusted_certs = True
args["firefox_profile"] = profile
args["capabilities"] = args["desired_capabilities"]
args.pop("desired_capabilities")
return args
def __init__(self):
self.profile = FirefoxProfile()
self.profile.set_preference('permissions.default.desktop-notification',
1)
if enable_change_proxy:
self.profile = self.change_proxy(host, port, self.profile)
else:
self.profile = self.clear_proxy(self.profile)
self.profile.update_preferences()
self.driver = webdriver.Firefox(firefox_profile=self.profile)
self.driver.maximize_window()
def setProxy(self, proxy):
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
profile.assume_untrusted_cert_issuer = True
prefix = "network.proxy."
profile.set_preference("%stype" % prefix, 1)
for type in ["http", "ssl", "ftp", "socks"]:
profile.set_preference("%s%s" % (prefix, type), proxy.getHost())
profile.set_preference("%s%s_port" % (prefix, type),
int(proxy.getPort()))
return profile
def __init__(self):
profile = FirefoxProfile()
profile.set_preference('media.volume_scale', '0.0') # Mute audio
self.driver = Firefox(firefox_profile=profile)
self.clients = None
self.players = None
self.join_link = None
self.state = GameState.STOPPED
self.quit_flag = False
def create_driver(browser="chrome"):
if browser == "chrome":
driver = webdriver.Chrome("./driver/chromedriver.exe")
else:
custom_profile = FirefoxProfile()
custom_profile.set_preference("dom.disable_beforeunload", True)
driver = webdriver.Firefox(firefox_profile=custom_profile, executable_path="./driver/geckodriver.exe")
driver.implicitly_wait(10)
return driver
def browser_setup():
"""Set a driver object."""
cfg = get_config()["browser"]
profile = FirefoxProfile()
user_agent = UserAgent()
profile.native_events_enabled = cfg["native_events_enabled"]
profile.set_preference("general.useragent.override", user_agent.random)
opts = FirefoxOptions()
opts.headless = cfg["headless"]
driver = Firefox(options=opts, executable_path=cfg["gecko_driver"])
driver.implicitly_wait(cfg["timeout"])
return driver
def set_browser(self, browser):
# CHROME
if browser == 'Chrome':
options = ChromeOptions();
options.add_argument("--disable-notifications");
self.browser = Chrome(options=options)
# FIREFOX
if browser == 'Firefox':
profile = FirefoxProfile();
profile.set_preference("dom.webnotifications.enabled", False);
self.browser = Firefox(firefox_profile=profile)
def test_add_extension_legacy_extension(capabilities, webserver):
current_directory = os.path.dirname(os.path.realpath(__file__))
root_directory = os.path.join(current_directory, '..', '..', '..', '..', '..')
extension_path = os.path.join(root_directory, 'third_party', 'firebug', 'firebug-1.5.0-fx.xpi')
profile = FirefoxProfile()
profile.add_extension(extension_path)
driver = Firefox(capabilities=capabilities, firefox_profile=profile)
profile_path = driver.firefox_profile.path
extension_path_in_profile = os.path.join(profile_path, 'extensions', '*****@*****.**')
assert os.path.exists(extension_path_in_profile)
driver.quit()
def test_add_extension_web_extension_without_id(capabilities, webserver):
current_directory = os.path.dirname(os.path.realpath(__file__))
root_directory = os.path.join(current_directory, '..', '..', '..', '..', '..')
extension_path = os.path.join(root_directory, 'third_party', 'firebug', 'mooltipass-1.1.87.xpi')
profile = FirefoxProfile()
profile.add_extension(extension_path)
driver = Firefox(capabilities=capabilities, firefox_profile=profile)
profile_path = driver.firefox_profile.path
extension_path_in_profile = os.path.join(profile_path, 'extensions', '[email protected]')
assert os.path.exists(extension_path_in_profile)
driver.quit()
def generate_profile(useragent="(default)"):
profile = FirefoxProfile()
if useragent.strip().lower()=="(default)":
status("Using the default useragent")
return profile
elif useragent.strip().lower()=="(random)":
random_useragent = generate_user_agent(os=('mac', 'linux'))
profile.set_preference("general.useragent.override", random_useragent) # To make our useragent random
status("Using random useragent "+random_useragent)
return profile
else:
profile.set_preference("general.useragent.override", useragent)
status("Using useragent "+useragent)
return profile
def setUp(self):
if _platform != "darwin":
self.display = Display(visible=0, size=(1280, 800))
self.display.start()
p = FirefoxProfile()
p.set_preference("webdriver.log.file", "/tmp/firefox_console")
self.driver = webdriver.Firefox(p)
self.base_url = os.getenv('SELENIUM_BASE_URL')
self.verificationErrors = []
self.accept_next_alert = True
self.driver.set_window_size(1280, 800)
self.project_name = "DBaaS"
def test_manual_proxy_is_set_in_profile():
profile = FirefoxProfile()
proxy = Proxy()
proxy.no_proxy = 'localhost, foo.localhost'
proxy.http_proxy = 'some.url:1234'
proxy.ftp_proxy = None
proxy.sslProxy = 'some2.url'
profile.set_proxy(proxy)
assert profile.default_preferences["network.proxy.type"] == ProxyType.MANUAL['ff_value']
assert profile.default_preferences["network.proxy.no_proxies_on"] == 'localhost, foo.localhost'
assert profile.default_preferences["network.proxy.http"] == 'some.url'
assert profile.default_preferences["network.proxy.http_port"] == 1234
assert profile.default_preferences["network.proxy.ssl"] == 'some2.url'
assert "network.proxy.ssl_port" not in profile.default_preferences
assert "network.proxy.ftp" not in profile.default_preferences
def test_add_extension_web_extension_with_id(capabilities, webserver):
current_directory = os.path.dirname(os.path.realpath(__file__))
root_directory = os.path.join(current_directory, '..', '..', '..', '..', '..')
# TODO: This file should probably live in a common directory.
extension_path = os.path.join(root_directory, 'javascript', 'node', 'selenium-webdriver',
'lib', 'test', 'data', 'firefox', 'webextension.xpi')
profile = FirefoxProfile()
profile.add_extension(extension_path)
driver = Firefox(capabilities=capabilities, firefox_profile=profile)
profile_path = driver.firefox_profile.path
extension_path_in_profile = os.path.join(profile_path, 'extensions', '*****@*****.**')
assert os.path.exists(extension_path_in_profile)
driver.get(webserver.where_is('simpleTest.html'))
driver.find_element_by_id('webextensions-selenium-example')
driver.quit()
def setup_firefox_profile(self):
"""Return a custom firefox profile with given preferences
and extensions.
"""
fp = FirefoxProfile()
if self.extensions:
# Load extensions.
for ext in self.extensions:
fp.add_extension(ext)
if self.preferences:
# Load preferences
for key, value in self.preferences:
fp.set_preference(key, value)
return fp
def test_that_unicode_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference('sample.preference.2', unicode('hi there'))
profile.update_preferences()
assert 'hi there' == profile.default_preferences["sample.preference.2"]
encoded = profile.encoded
decoded = base64.b64decode(encoded)
with BytesIO(decoded) as fp:
zip = zipfile.ZipFile(fp, "r")
for entry in zip.namelist():
if entry.endswith('user.js'):
user_js = zip.read(entry)
for line in user_js.splitlines():
if line.startswith(b'user_pref("sample.preference.2",'):
assert line.endswith(b'hi there");')
# there should be only one user.js
break
def __init__(self, should_hide_window, installation_path):
# ASSUMPTION: installation_path refers to a valid firefox executable.
firefox_binary = None if installation_path is None else FirefoxBinary(installation_path)
profile = FirefoxProfile()
profile.set_preference("intl.accept_languages", "en-us,en")
# do not load images:
profile.set_preference('browser.migration.version', 9001)
profile.set_preference('permissions.default.image', 2)
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
driver = selenium.webdriver.Firefox(firefox_profile=profile, firefox_binary=firefox_binary)
web_driver = EnhancedWebDriver(driver, should_hide_window)
super(FacebookScrapingWebDriver, self).__init__(web_driver, should_hide_window)
self.implicitly_wait(5)
def browser(request):
"""
initialize the selenium test case
"""
profile = FirefoxProfile()
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", DOWNLOAD_DIR)
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv")
b = Firefox(firefox_profile=profile)
b.implicitly_wait(10)
b.maximize_window()
request.addfinalizer(lambda *args: b.quit())
yield b
time.sleep(10)
def __init__(self,test,browser_name,url,test_data=None):
super(FrameworkTests,self).__init__(test)
self.test=test
self.browser_name=browser_name
self.url=url
self.driver=None
if self.browser_name=='firefox':
ffp = FirefoxProfile()
ffp.update_preferences()
self.driver = Firefox(firefox_profile=ffp)
elif self.browser_name=='chrome':
chromedriver = load_browser_driver("chromedriver")
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver=Chrome(chromedriver)
elif self.browser_name=='ie':
iedriver = load_browser_driver("IEDriverServer")
os.environ["webdriver.ie.driver"] = iedriver
self.driver=Ie(iedriver)
self.verification = []
self.verification.append("Test %s on browser %s" %(self.test,self.browser_name))
self.test_data=test_data
self.errors=[]
def setUp(self):
if os.environ.get("SELENIUM_HEADLESS"):
self.driver = webdriver.PhantomJS()
else:
self.profile = FirefoxProfile();
self.profile.set_preference("intl.accept_languages", "en")
self.driver = webdriver.Firefox(self.profile)
self.driver.set_window_size(1600, 1200)
self.driver.implicitly_wait(30)
self.base_url = "http://localhost:5000/"
self.verificationErrors = []
self.accept_next_alert = True
def load_cookie(self, session_id):
sessions = json.load(open( self.sessions_file ))
cookie_path = sessions[str(session_id)]["session_path"]
url = sessions[str(session_id)]["web_url"]
# Setting useragent to the same one the session saved with
useragent = sessions[str(session_id)]["useragent"]
profile = FirefoxProfile()
profile.set_preference("general.useragent.override", useragent )
cookies = pickle.load(open(cookie_path, "rb"))
try:
browser = Firefox(profile)
except:
error("Couldn't open browser to view session!")
return
browser.get(url)
browser.delete_all_cookies()
browser.execute_script("window.localStorage.clear()") # clear the current localStorage
for cookie in cookies:
browser.add_cookie(cookie)
status(f"Session {session_id} loaded")
browser.refresh()
self.browsers.append(browser)
def test_that_we_can_accept_a_profile(capabilities, webserver):
profile1 = FirefoxProfile()
profile1.set_preference("browser.startup.homepage_override.mstone", "")
profile1.set_preference("startup.homepage_welcome_url", webserver.where_is('simpleTest.html'))
profile1.update_preferences()
profile2 = FirefoxProfile(profile1.path)
driver = Firefox(
capabilities=capabilities,
firefox_profile=profile2)
title = driver.title
driver.quit()
assert "Hello WebDriver" == title
def __init__(self):
if config.USE_HEADLESS:
self.driver = webdriver.PhantomJS()
else:
self.profile = FirefoxProfile()
self.profile.set_preference("security.tls.version.fallback-limit", 0)
self.profile.set_preference("security.tls.version.min", 0)
self.driver = webdriver.Firefox(self.profile)
self.driver.set_window_size(1600, 1200)
self.driver.implicitly_wait(30)
self.verificationErrors = []
self.accept_next_alert = True
def firefox_profile(request):
profile = FirefoxProfile(request.config.getoption('firefox_profile'))
for preference in request.config.getoption('firefox_preferences'):
name, value = preference
if value.isdigit():
# handle integer preferences
value = int(value)
elif value.lower() in ['true', 'false']:
# handle boolean preferences
value = value.lower() == 'true'
profile.set_preference(name, value)
profile.update_preferences()
for extension in request.config.getoption('firefox_extensions'):
profile.add_extension(extension)
return profile
def start_firefox(proxy=None, user_agent=None):
p = FirefoxProfile("E://profiles")
p.accept_untrusted_certs = True
if user_agent:
p.set_preference("general.useragent.override", user_agent)
capabilities = DesiredCapabilities().FIREFOX.copy()
capabilities['acceptInsecureCerts'] = True
if proxy:
p.set_proxy(proxy)
browser = Firefox(firefox_profile=p, capabilities=capabilities)
return browser
def setUpClass(self):
# Starts a test deployment.
self.start_test_weblab_deployment()
if os.environ.get("SELENIUM_HEADLESS") and False:
self.driver = webdriver.PhantomJS()
else:
self.profile = FirefoxProfile()
self.profile.set_preference("intl.accept_languages", "en")
self.driver = webdriver.Firefox(self.profile)
self.driver.set_window_size(1400, 1000)
# self.driver.implicitly_wait(30)
self.base_url = "http://localhost:5000/"
self.verificationErrors = []
self.accept_next_alert = True
def __init__(self, url, retries, acknowledgements):
self.url = url
self.attempt = 0
self.retries = retries
self.acknowledgements = acknowledgements
self.proxy_test = google_prop.proxy_test
self.fp = FirefoxProfile()
#self.fp.set_preference('permissions.default.stylesheet', 2) # Disable css
self.fp.set_preference('permissions.default.image', 2) # Disable images
self.fp.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false') # Disable Flash
self.fp.set_preference('general.useragent_rotator.override', useragent.get_random_agent(google_prop.user_agent_list_url))
self.fp.set_preference("network.proxy.type", 1)
self.proxy_details = proxy.get_random_proxy(google_prop.proxy_list_url)
self.proxy = Proxy({
'httpProxy': self.proxy_details,
'sslProxy': self.proxy_details
})
pass
def firefox_profile(pytestconfig):
profile = None
if pytestconfig.getoption('firefox_profile'):
profile = FirefoxProfile(pytestconfig.getoption('firefox_profile'))
warnings.warn(
'--firefox-profile has been deprecated and will be removed in '
'a future release. Please use the firefox_options fixture to '
'set a profile path or FirefoxProfile object using '
'firefox_options.profile.', DeprecationWarning)
if pytestconfig.getoption('firefox_preferences'):
profile = profile or FirefoxProfile()
warnings.warn(
'--firefox-preference has been deprecated and will be removed in '
'a future release. Please use the firefox_options fixture to set '
'preferences using firefox_options.set_preference. If you are '
'using Firefox 47 or earlier then you will need to create a '
'FirefoxProfile object with preferences and set this using '
'firefox_options.profile.', DeprecationWarning)
for preference in pytestconfig.getoption('firefox_preferences'):
name, value = preference
if value.isdigit():
# handle integer preferences
value = int(value)
elif value.lower() in ['true', 'false']:
# handle boolean preferences
value = value.lower() == 'true'
profile.set_preference(name, value)
profile.update_preferences()
if pytestconfig.getoption('firefox_extensions'):
profile = profile or FirefoxProfile()
warnings.warn(
'--firefox-extensions has been deprecated and will be removed in '
'a future release. Please use the firefox_options fixture to '
'create a FirefoxProfile object with extensions and set this '
'using firefox_options.profile.', DeprecationWarning)
for extension in pytestconfig.getoption('firefox_extensions'):
profile.add_extension(extension)
return profile
def test_that_boolean_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference("sample.bool.preference", True)
profile.update_preferences()
assert profile.default_preferences["sample.bool.preference"] is True
def test_that_integer_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference("sample.int.preference", 12345)
profile.update_preferences()
assert 12345 == profile.default_preferences["sample.int.preference"]
def test_profiles_do_not_share_preferences():
profile1 = FirefoxProfile()
profile1.accept_untrusted_certs = False
profile2 = FirefoxProfile()
# Default is true. Should remain so.
assert profile2.default_preferences["webdriver_accept_untrusted_certs"] is True
class SeleniumBaseTest(unittest.TestCase):
def start_test_weblab_deployment(self):
# Load the configuration of the weblab instance that we have set up for just this test.
self.global_config = load_dir('test/deployments/webclient_dummy')
# Start the weblab instance. Because we have activated the dont-start flag it won't actually
# start listening on the port, but let us use the Flask test methods instead.
self.handler = self.global_config.load_process('myhost', 'myprocess')
self.core_server = GLOBAL_REGISTRY['mycore:myprocess@myhost']
@classmethod
def setUpClass(self):
# Starts a test deployment.
self.start_test_weblab_deployment()
if os.environ.get("SELENIUM_HEADLESS") and False:
self.driver = webdriver.PhantomJS()
else:
self.profile = FirefoxProfile()
self.profile.set_preference("intl.accept_languages", "en")
self.driver = webdriver.Firefox(self.profile)
self.driver.set_window_size(1400, 1000)
# self.driver.implicitly_wait(30)
self.base_url = "http://localhost:5000/"
self.verificationErrors = []
self.accept_next_alert = True
def is_element_present(self, how, what):
try: self.driver.find_element(by=how, value=what)
except NoSuchElementException, e: return False
return True
def is_alert_present(self):
try: self.driver.switch_to_alert()
except NoAlertPresentException, e: return False
return True
def close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally: self.accept_next_alert = True
@classmethod
def tearDownClass(self):
self.driver.quit()
self.assertEqual([], self.verificationErrors)
