def generate_firefox_profile(
self: "SeleniumTestability",
preferences: OptionalDictType = None,
accept_untrusted_certs: bool = False,
proxy: OptionalStrType = None,
) -> FirefoxProfile:
"""
Generates a firefox profile that sets up few required preferences for SeleniumTestability to support all necessary features.
Parameters:
- ``preferences`` - firefox profile preferences in dictionary format.
- ``accept_untrusted_certs`` should we accept untrusted/self-signed certificates.
- ``proxy`` proxy options
Note: If you opt out using this keyword, you are not able to get logs with ``Get Logs`` and Firefox.
"""
profile = FirefoxProfile()
if preferences:
for key, value in preferences.items(): # type: ignore
profile.set_preference(key, value)
profile.set_preference("devtools.console.stdout.content", True)
profile.accept_untrusted_certs = accept_untrusted_certs
if proxy:
profile.set_proxy(proxy)
profile.update_preferences()
return profile
def create_download_dir_profile_for_firefox(path_to_download,
mime_types_file=None,
*extensions_files):
"""
Example use
| ${profile} | Create Download Dir Profile For Firefox | Artifacts | Resources/mimeTypes.rdf | Resources/webdriver_element_locator-2.0-fx.xpi | Resources/selenium_ide-2.9.1-fx.xpi |
| Open Browser Extension | https://support.spatialkey.com/spatialkey-sample-csv-data/ | ff_profile_dir=${profile} |
| Click Element | //a[contains(@href,'sample.csv.zip')] |
"""
path_to_download_check = validate_create_artifacts_dir(
path_to_download)
fp = FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.manager.alertOnEXEOpen", False)
fp.set_preference("browser.download.dir", path_to_download_check)
fp.set_preference("xpinstall.signatures.required", False)
fp.set_preference("browser.helperApps.alwaysAsk.force", False)
fp.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"application/msword;application/csv;text/csv;image/png;image/jpeg;application/pdf;text/html;text/plain;application/octet-stream"
)
fp.set_preference("pdfjs.disabled", True)
fp.update_preferences()
for single_extension in extensions_files:
fp.add_extension(single_extension)
if mime_types_file is not None:
from shutil import copy2
copy2(os.path.normpath(mime_types_file), fp.profile_dir)
logger.info("Firefox Profile Created in dir '" + fp.profile_dir + "'")
return fp.profile_dir
def webDriverFirefox():
profile = FirefoxProfile()
profile.set_preference('font.default.x-western', 'sans-serif')
profile.set_preference('font.name.serif.x-western', 'sans-serif')
profile.set_preference('font.size.variable.x-western', 14)
profile.set_preference('general.warnOnAboutConfig', False)
profile.set_preference('layout.css.devPixelsPerPx', '1.0')
profile.set_preference('browser.sessionstore.resume_from_crash', False)
profile.set_preference('dom.webnotifications.enabled', False)
profile.update_preferences()
if system() == "FreeBSD":
binary = FirefoxBinary('/usr/local/bin/firefox')
elif system() == "Linux":
binary = FirefoxBinary('/usr/bin/firefox')
elif system() == "Windows":
pass
caps = DesiredCapabilities.FIREFOX.copy()
caps['marionette'] = True
caps['screenResolution'] = '2560x1440'
# marionette setting is fixed in selenium 3.0 and above by default
driver = webdriver.Firefox(firefox_profile=profile,
capabilities=caps,
firefox_binary=binary)
driver.set_window_size(1470, 900)
return driver
def test_proxy(proxy):
if not proxy:
return
# r = get('https://www.google.com/', proxy)
ip, port, version = proxy.split(',')
print(f'Checking proxy {ip}:{port}')
options = Options()
options.headless = True
profile = FirefoxProfile()
profile.set_preference('network.proxy.type', 1)
profile.set_preference("network.proxy.socks", ip)
profile.set_preference("network.proxy.socks_port", int(port))
profile.set_preference("network.proxy.socks_version", int(version))
profile.update_preferences()
browser = Firefox(executable_path=r"./geckodriver",
firefox_profile=profile,
options=options)
try:
browser.get('https://showip.net/')
soup = BeautifulSoup(browser.page_source, 'html.parser')
return soup.select_one('#checkip').get('value').strip() == proxy.split(
',')[0].strip()
except KeyboardInterrupt:
raise KeyboardInterrupt('Abort')
except:
logging.error(traceback.format_exc())
finally:
browser.quit()
def _setup_proxy(self, browser):
"""Setup proxy for Firefox or Chrome
:type browser: str
:param browser: Browser name
:rtype: selenium.webdriver.FirefoxProfile or selenium.webdriver.ChromeOptions
:returns: Proxy settings
"""
host = "127.0.0.1"
port = "9050"
if browser == "firefox":
firefox_profile = FirefoxProfile()
# Direct = 0, Manual = 1, PAC = 2, AUTODETECT = 4, SYSTEM = 5
firefox_profile.set_preference("network.proxy.type", 1)
firefox_profile.set_preference("network.proxy.socks", host)
firefox_profile.set_preference("network.proxy.socks_port", int(port))
firefox_profile.update_preferences()
return firefox_profile
elif browser == "chrome":
proxy = f"socks5://{host}:{port}"
chrome_options = ChromeOptions()
chrome_options.add_argument(f"--proxy-server={proxy}")
return chrome_options
else:
logger.info(f"No proxy setting was done for {browser}")
def get_firefox_driver():
profile = FirefoxProfile()
profile.set_preference('browser.cache.disk.enable', False)
profile.set_preference('browser.cache.memory.enable', False)
profile.set_preference('browser.cache.offline.enable', False)
profile.set_preference('network.cookie.cookieBehavior', 2)
if os.environ.get("DEV"):
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.socks", "127.0.0.1")
profile.set_preference("network.proxy.socks_port", 8050)
profile.set_preference("network.proxy.socks_version", 5)
profile.update_preferences()
caps = DesiredCapabilities().FIREFOX
caps["pageLoadStrategy"] = "none"
options = firefox_options()
options.headless = True
options.add_argument('start-maximized')
path = os.path.dirname(__file__)
if platform.system() == "Linux":
path = os.path.join(path, r"driver/geckodriver")
else:
path = os.path.join(path, r"driver/geckodriver_mac")
driver = webdriver.Firefox(executable_path=path,
firefox_options=options,
desired_capabilities=caps,
firefox_profile=profile)
driver.set_window_size(1920, 1080)
driver.implicitly_wait(1)
return driver
def create_download_dir_profile_for_firefox(path_to_download, mime_types_file=None, *extensions_files):
"""
Example use
| ${profile} | Create Download Dir Profile For Firefox | Artifacts | Resources/mimeTypes.rdf | Resources/webdriver_element_locator-2.0-fx.xpi | Resources/selenium_ide-2.9.1-fx.xpi |
| Open Browser Extension | https://support.spatialkey.com/spatialkey-sample-csv-data/ | ff_profile_dir=${profile} |
| Click Element | //a[contains(@href,'sample.csv.zip')] |
"""
path_to_download_check = validate_create_artifacts_dir(path_to_download)
fp = FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.manager.alertOnEXEOpen", False)
fp.set_preference("browser.download.dir", path_to_download_check)
fp.set_preference("xpinstall.signatures.required", False)
fp.set_preference("browser.helperApps.alwaysAsk.force", False)
fp.set_preference("browser.helperApps.neverAsk.saveToDisk",
"application/msword;application/csv;text/csv;image/png;image/jpeg;application/pdf;text/html;text/plain;application/octet-stream")
fp.set_preference("pdfjs.disabled", True)
fp.update_preferences()
for single_extension in extensions_files:
fp.add_extension(single_extension)
if mime_types_file is not None:
from shutil import copy2
copy2(os.path.normpath(mime_types_file), fp.profile_dir)
logger.info("Firefox Profile Created in dir '" + fp.profile_dir + "'")
return fp.profile_dir
def _proxy(self):
logger.info("proxy: {}".format(self.proxy))
if self.proxy and isinstance(self.proxy,
dict) and self.browser == 'chrome':
proxy = Proxy()
for k, v in self.proxy.items():
proxy.__setattr__(k, v)
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
return capabilities
elif self.proxy and isinstance(self.proxy,
dict) and self.browser == 'firefox':
profile = FirefoxProfile()
profile.set_preference('network.proxy.type', 1)
for k, v in self.proxy.items():
scheme = k.split("_")
host_details = v.split(":")
if len(scheme) > 1 and len(host_details) > 1:
profile.set_preference("network.proxy.%s" % scheme[0],
host_details[0])
profile.set_preference("network.proxy.%s_port" % scheme[0],
int(host_details[1]))
profile.update_preferences()
# profile.accept_untrusted_certs = True
return profile
def __init__(self):
profile = FirefoxProfile()
profile.set_preference("dom.webdriver.enabled", False)
profile.set_preference('useAutomationExtension', False)
profile.update_preferences()
desired = DesiredCapabilities.FIREFOX
super().__init__(firefox_profile=profile, desired_capabilities=desired)
def _connection():
o = Options()
fp = FirefoxProfile()
o.add_argument('-private')
fp.set_preference("network.proxy.type", 0)
#fp.set_preference("general.useragent.override","iPhone")
fp.update_preferences()
return Firefox(firefox_profile=fp, firefox_options=o)
def makeProfile(self, config):
profile = FirefoxProfile()
# Disable Firefox auto update
profile.set_preference("app.update.enabled", False)
try:
if config["fire_bug"]:
profile.add_extension(FIREBUG_EXTENSION)
domain = "extensions.firebug."
# Set default Firebug preferences
# Avoid Firebug start page
profile.set_preference(domain + "currentVersion", "2.0.6")
# Activate everything
profile.set_preference(domain + "allPagesActivation", "on")
profile.set_preference(domain + "defaultPanelName", "net")
# Enable Firebug on all sites
profile.set_preference(domain + "net.enableSites", True)
self.logger.info("Firebug profile settings enabled")
except KeyError:
self.logger.warning("Firebug profile settings failed")
pass
try:
if config["net_export"]:
profile.add_extension(NETEXPORT_EXTENSION)
# Set default NetExport preferences
self.har_output = config["net_export_output"]
#self.logger.info("Output HAR directory: {0}".format(self.har_output))
profile.set_preference(domain + "netexport.defaultLogDir", self.har_output)
profile.set_preference(domain + "netexport.autoExportToFile", True)
profile.set_preference(domain + "netexport.alwaysEnableAutoExport", True)
# Do not show preview output
profile.set_preference(domain + "netexport.showPreview", True)
profile.set_preference(domain + "netexport.pageLoadedTimeout", 3000)
# Log dir
self.logger.info("NetExport profile settings enabled.")
# Har ID to check file exists
self.har_id = int((datetime.datetime.now()-datetime.datetime(1970,1,1)).total_seconds()*1000000)
self.url += "/?{0}".format(self.har_id)
#print self.url
except KeyError:
self.logger.warning("NetExport profile settings failed")
pass
profile.update_preferences()
return profile
def clear_proxy(profile=None):
"""
:param profile:
:return:
"""
if profile is None:
profile = FirefoxProfile()
profile.set_preference("network.proxy.type", 0)
profile.update_preferences()
return profile
def test_that_we_can_accept_a_profile(capabilities, webserver):
profile1 = FirefoxProfile()
profile1.set_preference("browser.startup.homepage_override.mstone", "")
profile1.set_preference("startup.homepage_welcome_url",
webserver.where_is('simpleTest.html'))
profile1.update_preferences()
profile2 = FirefoxProfile(profile1.path)
driver = Firefox(capabilities=capabilities, firefox_profile=profile2)
title = driver.title
driver.quit()
assert "Hello WebDriver" == title
def private_connection(PROXY_HOST,PROXY_PORT):
fp = FirefoxProfile()
o = Options()
o.add_argument('-private')
fp.set_preference("network.proxy.type", 1)
fp.set_preference("network.proxy.http",PROXY_HOST)
fp.set_preference("network.proxy.http_port",int(PROXY_PORT))
fp.set_preference("network.proxy.ssl",PROXY_HOST)
fp.set_preference("network.proxy.ssl_port",int(PROXY_PORT))
fp.set_preference("general.useragent.override","iPhone")
fp.update_preferences()
return Firefox(firefox_profile=fp, firefox_options=o)
def create(self,
headless=False,
devtools=False,
incognito=False,
proxy_url='',
proxy_port=80):
"""Creates a Firefox browser.
:param headless: If the browser should start in headless mode
:type headless: bool
:param devtools: If the browser should start with opened dev tools
:type devtools: bool
:param incognito: If the browser should start in incognito mode
:type incognito: bool
:param proxy_url: The proxy url
:type proxy_url: str
:param proxy_port: The proxy port
:type proxy_port: int
:return: A fully prepared Firefox browser
:rtype: Firefox
"""
options = FirefoxOptions()
profile = FirefoxProfile()
options.profile = profile
profile.set_preference("devtools.jsonview.enabled",
False) # To parse JSON
profile.set_preference("devtools.inspector.show-three-pane-tooltip",
False) # Monopolizes focus
# profile.set_preference("devtools.inspector.three-pane-first-run", False) # Monopolizes focus
if headless:
options.headless = True
self._log_headless()
if devtools:
options.add_argument('-devtools')
self._log_devtools()
if incognito:
profile.set_preference("browser.privatebrowsing.autostart", True)
self._log_incognito()
if not proxy_url == '':
profile.set_preference("network.proxy.type",
ProxyType.MANUAL['ff_value'])
profile.set_preference("network.proxy.http", proxy_url)
profile.set_preference("network.proxy.http_port", proxy_port)
profile.set_preference("network.proxy.share_proxy_settings", True)
proxy_address = "{}:{}".format(proxy_url, proxy_port)
self._log_proxy(proxy_address)
profile.update_preferences()
self.log.debug("Arguments: {}".format(options.arguments))
if platform.system() == 'Windows':
return Firefox(options=options, executable_path='geckodriver.exe')
else:
return Firefox(options=options)
def test_that_we_can_accept_a_profile(capabilities, webserver):
profile1 = FirefoxProfile()
profile1.set_preference("browser.startup.homepage_override.mstone", "")
profile1.set_preference("startup.homepage_welcome_url", webserver.where_is('simpleTest.html'))
profile1.update_preferences()
profile2 = FirefoxProfile(profile1.path)
driver = Firefox(
capabilities=capabilities,
firefox_profile=profile2)
title = driver.title
driver.quit()
assert "Hello WebDriver" == title
def firefox_profile(request):
profile = FirefoxProfile(request.config.getoption('firefox_profile'))
for preference in request.config.getoption('firefox_preferences'):
name, value = preference
if value.isdigit():
# handle integer preferences
value = int(value)
elif value.lower() in ['true', 'false']:
# handle boolean preferences
value = value.lower() == 'true'
profile.set_preference(name, value)
profile.update_preferences()
for extension in request.config.getoption('firefox_extensions'):
profile.add_extension(extension)
return profile
def profile(user_agent: Optional[str] = None,
language: str = 'en-us',
private: bool = False,
disable_images: bool = False,
mute_audio: bool = False,
proxy: Optional[Proxy] = None,
path: Optional[str] = None) -> FirefoxProfile:
profile = FirefoxProfile(
path if path and os.path.exists(path) else None)
if user_agent:
profile.set_preference('general.useragent.override', user_agent)
if language:
profile.set_preference('intl.accept_languages', language)
if private:
profile.set_preference('browser.privatebrowsing.autostart', True)
if disable_images:
profile.set_preference('permissions.default.image', 2)
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
False)
if mute_audio:
profile.set_preference('media.volume_scale', '0.0')
if proxy:
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.http', proxy.host)
profile.set_preference('network.proxy.http_port', proxy.port)
profile.set_preference('network.proxy.ssl', proxy.host)
profile.set_preference('network.proxy.ssl_port', proxy.port)
profile.set_preference('network.proxy.ftp', proxy.host)
profile.set_preference('network.proxy.ftp_port', proxy.port)
profile.set_preference('network.proxy.socks', proxy.host)
profile.set_preference('network.proxy.socks_port', proxy.port)
profile.set_preference('network.proxy.socks_version', 5)
profile.set_preference('signon.autologin.proxy', True)
profile.set_preference('marionatte', False)
profile.set_preference('dom.webdriver.enabled', False)
profile.set_preference('media.peerconnection.enabled', False)
profile.set_preference('useAutomationExtension', False)
profile.set_preference('general.warnOnAboutConfig', False)
profile.update_preferences()
return profile
def firefox_webdriver_factory(
accept_language: str = 'en-us, en') -> FirefoxWebDriver:
"""
Create a firefox webdriver, in headless mode or not, depending on the setting HEADLESS_TEST_MODE
"""
options = FirefoxOptions()
options.headless = settings.HEADLESS_TEST_MODE
profile = FirefoxProfile()
profile.set_preference('intl.accept_languages', accept_language)
profile.update_preferences()
selenium = FirefoxWebDriver(options=options, firefox_profile=profile)
selenium.implicitly_wait(15)
return selenium
def create_driver(debug: bool) -> WebDriver:
firefox_options = Options()
if not debug:
firefox_options.add_argument("-headless")
# disable navigator.webdriver in order to make driver automation
# undetectable: https://stackoverflow.com/a/60626696
profile = FirefoxProfile()
profile.set_preference("dom.webdriver.enabled", False)
profile.set_preference("useAutomationExtension", False)
profile.update_preferences()
driver = Firefox(
executable_path="./selenium-drivers/geckodriver",
options=firefox_options,
firefox_profile=profile,
)
return driver
def firefox_profile(pytestconfig):
profile = None
if pytestconfig.getoption("firefox_profile"):
profile = FirefoxProfile(pytestconfig.getoption("firefox_profile"))
warnings.warn(
"--firefox-profile has been deprecated and will be removed in "
"a future release. Please use the firefox_options fixture to "
"set a profile path or FirefoxProfile object using "
"firefox_options.profile.",
DeprecationWarning,
)
if pytestconfig.getoption("firefox_preferences"):
profile = profile or FirefoxProfile()
warnings.warn(
"--firefox-preference has been deprecated and will be removed in "
"a future release. Please use the firefox_options fixture to set "
"preferences using firefox_options.set_preference. If you are "
"using Firefox 47 or earlier then you will need to create a "
"FirefoxProfile object with preferences and set this using "
"firefox_options.profile.",
DeprecationWarning,
)
for preference in pytestconfig.getoption("firefox_preferences"):
name, value = preference
if value.isdigit():
# handle integer preferences
value = int(value)
elif value.lower() in ["true", "false"]:
# handle boolean preferences
value = value.lower() == "true"
profile.set_preference(name, value)
profile.update_preferences()
if pytestconfig.getoption("firefox_extensions"):
profile = profile or FirefoxProfile()
warnings.warn(
"--firefox-extensions has been deprecated and will be removed in "
"a future release. Please use the firefox_options fixture to "
"create a FirefoxProfile object with extensions and set this "
"using firefox_options.profile.",
DeprecationWarning,
)
for extension in pytestconfig.getoption("firefox_extensions"):
profile.add_extension(extension)
return profile
def test_that_unicode_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference('sample.preference.2', unicode('hi there'))
profile.update_preferences()
assert 'hi there' == profile.default_preferences["sample.preference.2"]
encoded = profile.encoded
decoded = base64.b64decode(encoded)
with BytesIO(decoded) as fp:
zip = zipfile.ZipFile(fp, "r")
for entry in zip.namelist():
if entry.endswith('user.js'):
user_js = zip.read(entry)
for line in user_js.splitlines():
if line.startswith(b'user_pref("sample.preference.2",'):
assert line.endswith(b'hi there");')
# there should be only one user.js
break
def test_that_unicode_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference('sample.preference.2', unicode('hi there'))
profile.update_preferences()
assert 'hi there' == profile.default_preferences["sample.preference.2"]
encoded = profile.encoded
decoded = base64.b64decode(encoded)
with BytesIO(decoded) as fp:
zip = zipfile.ZipFile(fp, "r")
for entry in zip.namelist():
if entry.endswith('user.js'):
user_js = zip.read(entry)
for line in user_js.splitlines():
if line.startswith(b'user_pref("sample.preference.2",'):
assert line.endswith(b'hi there");')
# there should be only one user.js
break
def generate_firefox_profile(
self: "SeleniumTestability",
options: OptionalDictType = None,
accept_untrusted_certs: bool = False,
proxy: OptionalStrType = None,
) -> FirefoxProfile:
profile = FirefoxProfile()
if options:
for key, value in options.items(): # type: ignore
profile.set_preference(key, value)
profile.set_preference("devtools.console.stdout.content", True)
if accept_untrusted_certs:
profile.accept_untrusted_certs
if proxy:
profile.set_proxy(proxy)
profile.update_preferences()
return profile
def __init__(self,test,browser_name,url,test_data=None):
super(FrameworkTests,self).__init__(test)
self.test=test
self.browser_name=browser_name
self.url=url
self.driver=None
if self.browser_name=='firefox':
ffp = FirefoxProfile()
ffp.update_preferences()
self.driver = Firefox(firefox_profile=ffp)
elif self.browser_name=='chrome':
chromedriver = load_browser_driver("chromedriver")
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver=Chrome(chromedriver)
elif self.browser_name=='ie':
iedriver = load_browser_driver("IEDriverServer")
os.environ["webdriver.ie.driver"] = iedriver
self.driver=Ie(iedriver)
self.verification = []
self.verification.append("Test %s on browser %s" %(self.test,self.browser_name))
self.test_data=test_data
self.errors=[]
def firefox_profile(pytestconfig):
profile = None
if pytestconfig.getoption('firefox_profile'):
profile = FirefoxProfile(pytestconfig.getoption('firefox_profile'))
warnings.warn(
'--firefox-profile has been deprecated and will be removed in '
'a future release. Please use the firefox_options fixture to '
'set a profile path or FirefoxProfile object using '
'firefox_options.profile.', DeprecationWarning)
if pytestconfig.getoption('firefox_preferences'):
profile = profile or FirefoxProfile()
warnings.warn(
'--firefox-preference has been deprecated and will be removed in '
'a future release. Please use the firefox_options fixture to set '
'preferences using firefox_options.set_preference. If you are '
'using Firefox 47 or earlier then you will need to create a '
'FirefoxProfile object with preferences and set this using '
'firefox_options.profile.', DeprecationWarning)
for preference in pytestconfig.getoption('firefox_preferences'):
name, value = preference
if value.isdigit():
# handle integer preferences
value = int(value)
elif value.lower() in ['true', 'false']:
# handle boolean preferences
value = value.lower() == 'true'
profile.set_preference(name, value)
profile.update_preferences()
if pytestconfig.getoption('firefox_extensions'):
profile = profile or FirefoxProfile()
warnings.warn(
'--firefox-extensions has been deprecated and will be removed in '
'a future release. Please use the firefox_options fixture to '
'create a FirefoxProfile object with extensions and set this '
'using firefox_options.profile.', DeprecationWarning)
for extension in pytestconfig.getoption('firefox_extensions'):
profile.add_extension(extension)
return profile
def init_driver_firefox(self, total_width=1600, total_height=900):
options = Options()
fp = FirefoxProfile()
if self.headless:
options.add_argument("--headless")
if not self.download_images:
fp.set_preference('permissions.default.image', 2)
fp.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
if self.proxy_host and self.proxy_port:
fp = webdriver.FirefoxProfile()
# network.proxy.type: Direct = 0, Manual = 1, PAC = 2, AUTODETECT = 4, SYSTEM = 5
fp.set_preference("network.proxy.type", 1)
fp.set_preference("network.proxy.http", self.proxy_host)
fp.set_preference("network.proxy.http_port", int(self.proxy_port)) # WARNING ! proxy_port should be int
fp.set_preference("general.useragent.override", self.user_agent)
fp.update_preferences()
elif self.proxy_port or self.proxy_host:
raise(Exception('If you want to use a proxy, please provide proxy_host and proxy_port'))
self.driver = Firefox(firefox_options=options, firefox_profile=fp)
self.driver.set_window_size(total_width, total_height)
class function_webdriver():
def __init__(self):
self.profile = FirefoxProfile()
self.profile.set_preference('permissions.default.desktop-notification',
1)
if enable_change_proxy:
self.profile = self.change_proxy(host, port, self.profile)
else:
self.profile = self.clear_proxy(self.profile)
self.profile.update_preferences()
self.driver = webdriver.Firefox(firefox_profile=self.profile)
self.driver.maximize_window()
def change_proxy(self, host, port, profile: object = None):
if profile is None:
profile = FirefoxProfile()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", host)
profile.set_preference("network.proxy.http_port", int(port))
profile.set_preference("network.proxy.ssl", host)
profile.set_preference("network.proxy.ssl_port", int(port))
profile.update_preferences()
return profile
def clear_proxy(self, profile=None):
if profile is None:
profile = FirefoxProfile()
profile.set_preference("network.proxy.type", 0)
profile.update_preferences()
return profile
def login(self, user_name, password):
print("Opened facebook")
username_box = self.driver.find_element_by_id('email')
username_box.send_keys(user_name)
print("Email Id entered")
time.sleep(1)
password_box = self.driver.find_element_by_id('pass')
password_box.send_keys(password)
print("Password entered")
login_box = self.driver.find_element_by_id('loginbutton')
login_box.click()
print("Done")
return True
def logout(self):
logout = self.driver.find_element_by_id("userNavigationLabel")
logout.click()
time.sleep(1)
logout2 = self.driver.find_element_by_css_selector(
"li._54ni:nth-child(12) > a:nth-child(1) > span:nth-child(1) > span:nth-child(1)"
)
logout2.click()
return True
def get_URL(self, url):
return self.driver.get(url)
def quit(self):
return self.clear_proxy(self.profile)
#def wait_for_page_load(self):
# try:
# element_present = EC.presence_of_element_located((By.ID, "element_id"))
# WebDriverWait(self.driver, 10).until(element_present)
# except TimeoutException:
# print("Timed out waiting for page to load")
def scroll_page(self):
lenOfPage = self.driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;"
)
#self.wait_for_page_load()
try:
#element_present = EC.presence_of_element_located((By.ID, "element_id"))
element_present = EC.presence_of_element_located(
(By.TAG_NAME, "html"))
WebDriverWait(self.driver, 10).until(element_present)
except TimeoutException:
print("Timed out waiting for page to load")
match = False
while (match == False):
lastCount = lenOfPage
time.sleep(2)
lenOfPage = self.driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;"
)
if lastCount == lenOfPage:
match = True
def get_URL_image(self):
images = self.driver.find_elements_by_css_selector(
"a[rel='theater'][data-render-location='homepage_stream']")
#images = self.driver.find_elements_by_css_selector("a[class='_5pcq']")
#images = self.driver.find_elements_by_class_name("_5pcq")
for image in images:
if ("video" in image.get_attribute("href")):
continue
str = image.get_attribute("href")
print(str)
self.click_Image(image)
try:
# element_present = EC.presence_of_element_located((By.ID, "element_id"))
element_present = EC.presence_of_element_located(
(By.CLASS_NAME, "b=[spotlight]"))
WebDriverWait(self.driver, 10).until(element_present)
except TimeoutException:
print("Timed out waiting for page to load")
wait = WebDriverWait(self.driver, 10)
get_like_share = wait.until(
EC.presence_of_element_located(
(By.ID, "fbPhotoSnowliftFeedback")))
# Count the number of likes
try:
likes = get_like_share.find_element_by_class_name('_4arz')
except NoSuchElementException as e:
print("The number of likes: 0")
pass
else:
number_of_likes = likes.get_attribute("innerText")
print('The number of likes:', number_of_likes)
# Count the number of shares
try:
shares = get_like_share.find_element_by_class_name(
'UFIShareLink')
except (NoSuchElementException):
print("The number of shares: 0")
pass
else:
number_of_shares = shares.get_attribute("innerText")
print('The number of shares:', number_of_shares)
return image
def click_Image(self, image):
self.driver.execute_script("arguments[0].scrollIntoView(true);", image)
self.driver.execute_script("arguments[0].click();", image)
def test_that_boolean_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference("sample.bool.preference", True)
profile.update_preferences()
assert profile.default_preferences["sample.bool.preference"] is True
def test_that_integer_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference("sample.int.preference", 12345)
profile.update_preferences()
assert 12345 == profile.default_preferences["sample.int.preference"]
def __init__(
self,
# webdriver.Firefox parameters start here
firefox_profile: webdriver.FirefoxProfile = webdriver.
FirefoxProfile(),
firefox_binary=None,
timeout=30,
capabilities=None,
proxy=None,
executable_path="geckodriver",
options=None,
service_log_path="geckodriver.log",
firefox_options=None,
service_args=None,
desired_capabilities=None,
log_path=None,
keep_alive=True,
# CamouflageHandler parameters
user_agents: list = [],
allow_reuse_ip_after: int = 10,
time_between_calls: float = 0.25,
random_delay: bool = True,
# Parameters of this class
change_ip_after: int = 42,
change_user_agent_after: int = 0,
cookie_domain: str = '',
persist_cookies_when_close: bool = False,
reload_cookies_when_start: bool = False,
location_of_cookies: str = 'cookies.pkl'):
"""Starts a new session of TorFirefoxWebdriver.
Keyword arguments:
change_ip_after -- Number of calls before changing the IP. (default 42)
change_user_agent_after -- Number of calls before changing the user-agent. If the number is zero, the user-agent never will be changed (default 0)
cookie_domain -- Domain of the cookie, eg., https://example.com/ (default '')
persist_cookies_when_close -- Whether to save cookies when closing the driver (default False)
reload_cookies_when_start -- Whether to reload the cookies that were saved when closing the last driver session (default False)
location_of_cookies -- Where to save cookies (default 'cookies.pkl')
"""
CamouflageHandler.__init__(self,
user_agents=user_agents,
allow_reuse_ip_after=allow_reuse_ip_after,
time_between_calls=time_between_calls,
random_delay=random_delay)
self.number_of_requests_made = 0
self.change_ip_after = change_ip_after
self.change_user_agent_after = change_user_agent_after
firefox_profile.set_preference("network.proxy.type", 1)
firefox_profile.set_preference("network.proxy.socks", '127.0.0.1')
firefox_profile.set_preference("network.proxy.socks_port", 9050)
firefox_profile.update_preferences()
webdriver.Firefox.__init__(self,
firefox_profile=firefox_profile,
firefox_binary=firefox_binary,
timeout=timeout,
capabilities=capabilities,
proxy=proxy,
executable_path=executable_path,
options=options,
service_log_path=service_log_path,
firefox_options=firefox_options,
service_args=service_args,
desired_capabilities=desired_capabilities,
log_path=log_path,
keep_alive=keep_alive)
# Enables advanced Firefox preferences (to change user-agents)
if change_user_agent_after > 0:
if len(user_agents) > 0:
super().get('about:config')
self.find_element_by_id('showWarningNextTime').click()
self.find_element_by_id('warningButton').click()
else:
raise EmptyUserAgentListError(
'You are trying to rotate user-agent with an empty list of user-agents.'
)
self.cookie_domain = cookie_domain
self.persist_cookies_when_close = persist_cookies_when_close
self.location_of_cookies = location_of_cookies
if reload_cookies_when_start:
if not self.cookie_domain:
raise CookieDomainError(
'To reload cookies, you need to specify their domain')
self.reload_cookies()
def ClearProxy(self, profile=None):
if profile is None:
profile = FirefoxProfile()
profile.set_preference("network.proxy.type", 0)
profile.update_preferences()
return profile
def test_that_boolean_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference("sample.bool.preference", True)
profile.update_preferences()
assert profile.default_preferences["sample.bool.preference"] is True
def test_that_integer_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference("sample.int.preference", 12345)
profile.update_preferences()
assert 12345 == profile.default_preferences["sample.int.preference"]
# p.set_preference("browser.download.panel.shown", False) ok
p.set_preference("browser.shell.checkDefaultBrowser", False)
p.set_preference("browser.download.folderList", 0)
p.set_preference("browser.download.manager.showWhenStarting", False)
p.set_preference("browser.download.manager.alertOnEXEOpen", False)
# Só downloads PDF
p.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/pdf")
# p.set_preference("browser.download.importedFromSqlite", False) ok
# p.set_preference("pdfjs.disabled", True) ok
# p.set_preference("pdfjs.showPreviousViewOnLoad", False) ok
# p.set_preference("pdfjs.previousHandler.preferredAction", 4) ok
# p.set_preference("pdfjs.previousHandler.alwaysAskBeforeHandling", True)
p.set_preference("plugin.disable_full_page_plugin_for_types", "application/pdf")
p.set_preference("pdfjs.migrationVersion", 2)
#Configurações Câmera e audio
# p.set_preference("media.webspeech.recognition.enable", True)
p.set_preference("camera.control.face_detection.enabled", True)
p.set_preference("dom.imagecapture.enabled", True)
# Limpa cache e cookies
p.set_preference('browser.cache.disk.enable', False)
p.set_preference('browser.cache.memory.enable', False)
p.set_preference('browser.cache.offline.enable', False)
p.set_preference('network.cookie.cookieBehavior', 2)
p.update_preferences() #Provavelmente aplica
driver = webdriver.Firefox(firefox_binary="caminho/firefox36", firefox_profile=pp)
class Scrapper():
UserAgentList = ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:63.0) Gecko/20100101 Firefox/63.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:75.0) Gecko/20100101 Firefox/75.0",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0",
"Mozilla/5.0 (Windows NT 10.0; rv:68.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"
]
def __init__(self,
name,
urls,
str_file,
test_url = "http://ipv4.plain-text-ip.com/",
#test_url = "https://www.york.ac.uk/teaching/cws/wws/webpage1.html",
data_folder = "{}/scrapped".format(thispath),
proxy = False,
headless= False,
wait=1,
max_nexts=0):
logger.info("")
logger.info("********************************************************************")
logger.info("*** Inicializando el scrapper ***")
logger.info("********************************************************************")
logger.info("")
self.name = name
self.urls = urls
self.str_file = str_file
self.test_url = test_url
self.data_folder = "{}/{}".format(data_folder,self.name)
self.proxy = proxy
if self.proxy:
self.actual_proxy = False
if isinstance(self.proxy,dict):
try:
self.proxy_list = proxies(typ=self.proxy["Types"],
lim=self.proxy["Limit"],
countries_list=self.proxy["Countries"])
except:
raise
else:
self.proxy_list = proxies()
else:
self.proxy_list = False
self.headless = headless
self.wait = wait
self.max_nexts = max_nexts
self.data = []
self.previus_data = []
self.tread = False
self.set_dom()
self.configure_driver()
self.crawl()
def set_dom(self):
dom_strc = dom(self.str_file)
self.main_strc = dom_strc.main_strc
self.data_strc = dom_strc.data_strc
def configure_driver(self):
self.firefox_capabilities = DesiredCapabilities.FIREFOX
UserAgent = random.choice(self.UserAgentList)
logger.info("User agent: {}".format(UserAgent))
if self.headless:
self.options = Options()
self.options.add_argument('--headless')
self.firefox_profile = FirefoxProfile()
self.firefox_profile.set_preference("browser.privatebrowsing.autostart", True)
self.firefox_profile.set_preference("general.useragent.override", UserAgent)
self.firefox_profile.update_preferences()
else:
self.options = Options()
self.firefox_profile = FirefoxProfile()
self.firefox_profile.set_preference("browser.privatebrowsing.autostart", True)
# Esta lineas no sirven por ahora, buscar alternativas
"""self.firefox_profile.set_preference('permissions.default.image',2)
self.firefox_profile.set_preference("permissions.default.stylesheet",2)"""
self.firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', False)
self.firefox_profile.set_preference('media.navigator.video.enabled',False)
self.firefox_profile.set_preference('media.encoder.webm.enabled',False)
self.firefox_profile.set_preference('media.ffmpeg.enabled',False)
self.firefox_profile.set_preference('media.flac.enabled',False)
self.firefox_profile.set_preference("general.useragent.override", UserAgent)
self.firefox_profile.update_preferences()
try:
self.driver = Firefox(capabilities=self.firefox_capabilities,
options=self.options,
firefox_profile=self.firefox_profile,
executable_path='geckodriver')
except:
raise
def createDataFile(self):
try:
if not path.isdir(self.data_folder):
makedirs(self.data_folder)
if not path.isfile(self.data_file):
with open(self.data_file, "w") as datafile:
writer = csv.DictWriter(datafile, delimiter="\t", fieldnames=list(self.data_strc.keys()))
writer.writeheader()
except:
raise
def crawl(self):
logger.info("Starting crawler")
try:
for enum, url in enumerate(self.urls):
logger.info("Crawling {}".format(url["URL"]))
filename = url["PARAM"].replace(" ","")
self.data_file = "{}/{}.csv".format(self.data_folder,filename)
self.createDataFile()
self.url = url
self.navigate()
self.driver.close()
logger.info("Finishing crawler")
except:
raise
def navigate(self):
try:
self.set_proxy()
self.driver.get(self.url["URL"])
self.check_containers()
self.get_data()
self.nexts = 0
self.next()
except:
self.navigate()
def check_containers(self):
try:
print("---> Check containers")
WebDriverWait(self.driver, 30).until(expected.visibility_of_element_located((By.CSS_SELECTOR, '{}'.format(self.main_strc["LiCont"]["Selector"]))))
self.main_strc["LiCont"]["Elements"] = self.driver.find_elements_by_css_selector('{}'.format(self.main_strc["LiCont"]["Selector"]))
self.main_strc["ElCont"]["Elements"] = []
if self.main_strc["LiCont"]["Elements"] != []:
for element in self.main_strc["LiCont"]["Elements"]:
self.main_strc["ElCont"]["Elements"] += element.find_elements_by_css_selector('{}'.format(self.main_strc["ElCont"]["Selector"]))
if self.actual_proxy:
self.proxy_list.proxy_notwork(self.actual_proxy.ip, self.actual_proxy.port)
except TimeoutException as e:
print("---> Nos cacharon")
if self.actual_proxy:
self.proxy_list.proxy_notwork(self.actual_proxy.host, self.actual_proxy.port)
self.set_proxy()
try:
self.driver.close()
self.navigate(self)
except:
self.navigate(self)
pass
except WebDriverException as e:
print("---> Ni idea que pasó")
if self.actual_proxy:
self.proxy_list.proxy_notwork(self.actual_proxy.ip, self.actual_proxy.port)
verified_proxy_host = ""
self.set_proxy()
try:
self.driver.close()
self.navigate(self)
except:
self.navigate(self)
pass
def get_data(self):
try:
if self.main_strc["ElCont"]["Elements"] != []:
for element in self.main_strc["ElCont"]["Elements"]:
data_dict = {}
for key, selector in self.data_strc.items():
elements = element.find_elements_by_css_selector('{}'.format(selector["Selector"]))
data_dict[key] = ""
if key == "Link":
data_dict[key] += elements[0].get_attribute('href')
else:
if len(elements) > 1:
for e in elements:
data_dict[key] += "<e>"+e.text+"</e>"
elif len(elements)==1:
data_dict[key] = elements[0].text
with open(self.data_file, "a") as datafile:
if "".join(list(data_dict.values())) != "":
writer = csv.DictWriter(datafile, delimiter="\t", fieldnames=list(self.data_strc.keys()))
writer.writerow(data_dict)
except:
raise
def next(self):
if self.main_strc["Next"]["Selector"]:
self.nexts += 1
#print(self.max_nexts, self.nexts, self.max_nexts == 0 or self.nexts < self.max_nexts)
if self.max_nexts == 0 or self.nexts < self.max_nexts:
try:
self.main_strc["Next"]["Elements"] = WebDriverWait(self.driver,2).until(expected.element_to_be_clickable((By.CSS_SELECTOR, '{}'.format(self.main_strc["Next"]["Selector"]))))
wait = random.randrange(5,10)
sleep(wait)
self.main_strc["Next"]["Elements"].click()
self.check_containers()
self.get_data()
self.next()
except:
raise
def set_proxy(self):
if self.proxy_list:
self.proxy_list.select_proxy()
self.actual_proxy = self.proxy_list.selected_proxy
logger.info("Setting {}:{} from {} as proxy".format(self.actual_proxy.host,
self.actual_proxy.port,
self.actual_proxy.geo))
self.driver.execute("SET_CONTEXT", {"context": "chrome"})
try:
self.driver.execute_script("""
Services.prefs.setIntPref('network.proxy.type', 1);
Services.prefs.setCharPref("network.proxy.http", arguments[0]);
Services.prefs.setIntPref("network.proxy.http_port", arguments[1]);
Services.prefs.setCharPref("network.proxy.ssl", arguments[0]);
Services.prefs.setIntPref("network.proxy.ssl_port", arguments[1]);""",
self.actual_proxy.host,self.actual_proxy.port
)
finally:
self.driver.execute("SET_CONTEXT", {"context": "content"})
self.test_proxy()
else:
logger.info("------------------------La cagó pedazo e bola!!!!")
"""
Services.prefs.setIntPref('network.proxy.type', 1);
Services.prefs.setCharPref("network.proxy.http", arguments[0]);
Services.prefs.setIntPref("network.proxy.http_port", arguments[1]);
Services.prefs.setCharPref("network.proxy.ssl", arguments[0]);
Services.prefs.setIntPref("network.proxy.ssl_port", arguments[1]);
Services.prefs.setCharPref("network.proxy.ftp", arguments[0]);
Services.prefs.setIntPref("network.proxy.ftp_port", arguments[1]);
"""
def test_proxy(self, timeout = 10):
if self.actual_proxy:
logger.info("Testing {}:{} from {} as proxy".format(self.actual_proxy.host))
try:
self.driver.set_page_load_timeout(timeout)
self.driver.get(self.test_url)
logger.info("{} worked in test page".format(self.actual_proxy.host))
except TimeoutException as e:
if timeout == 10:
test_proxy(self, timeout = 20)
else:
logger.info("{} Time out".format(self.actual_proxy.host))
self.proxy_list.proxy_notwork(self.actual_proxy.host self.actual_proxy.port)
self.set_proxy()
except WebDriverException as e:
logger.info("{} Something goes wrong".format(self.actual_proxy.host))
self.proxy_list.proxy_notwork(self.actual_proxy.ip, self.actual_proxy.port)
self.set_proxy()
def profile():
profile = FirefoxProfile()
profile.set_preference('browser.startup.homepage_override.mstone', '')
profile.set_preference('startup.homepage_welcome_url', 'about:')
profile.update_preferences()
return profile
