def test_profiles_do_not_share_preferences():
profile1 = FirefoxProfile()
profile1.accept_untrusted_certs = False
profile2 = FirefoxProfile()
# Default is true. Should remain so.
assert profile2.default_preferences[
"webdriver_accept_untrusted_certs"] is True
def prepare_browsers(headless: bool, driver_path: str,
twitter_profile_path: str) -> Browsers:
"""
Sets up browsers to search accounts
:param headless bool: Should search be performed in headless mode
:param driver_path: Path to geckodriver
:param twitter_profile_path: Path to twitter profile folder
:return: tuple of browsers, that are logged in LinkedIn and Xing
"""
logging.info("Running Twitter scraper from profile in %s",
twitter_profile_path)
driver_path = driver_path if driver_path else "geckodriver"
profile = FirefoxProfile()
twitter_profile = FirefoxProfile(twitter_profile_path)
twitter_profile.DEFAULT_PREFERENCES["frozen"][
"extensions.autoDisableScopes"] = 0
twitter_profile.set_preference("extensions.enabledScopes", 15)
logins = social_media_logins(driver_path, profile)
driver_options = FirefoxOptions()
driver_options.headless = headless
linked_in_driver = Firefox(options=driver_options,
firefox_profile=profile,
executable_path=driver_path)
xing_driver = Firefox(options=driver_options,
firefox_profile=profile,
executable_path=driver_path)
twitter_driver = Firefox(options=driver_options,
firefox_profile=twitter_profile,
executable_path=driver_path)
set_login_data(linked_in_driver, logins[0])
set_login_data(xing_driver, logins[1])
retoggleAllTheAddons(twitter_driver)
return Browsers(linked_in_driver, xing_driver, twitter_driver)
def test_that_we_can_accept_a_profile(capabilities, webserver):
profile1 = FirefoxProfile()
profile1.set_preference("browser.startup.homepage_override.mstone", "")
profile1.set_preference("startup.homepage_welcome_url",
webserver.where_is('simpleTest.html'))
profile1.update_preferences()
profile2 = FirefoxProfile(profile1.path)
driver = Firefox(capabilities=capabilities, firefox_profile=profile2)
title = driver.title
driver.quit()
assert "Hello WebDriver" == title
def run_local(self, browser_version):
"""
Return the local driver
: TODO Add support to other browsers
"""
if command_line.get_browser().lower() == "chrome":
options = chromeOptions()
options.add_argument("disable-infobars")
if command_line.headless() == "y":
options.add_argument("--headless") # For Headless chrome
caps = DesiredCapabilities.CHROME
local_driver = webdriver.Chrome(options=options,
desired_capabilities=caps)
return local_driver
if command_line.get_browser().lower() == "ff" \
or command_line.get_browser().lower() == "firefox":
profile = FirefoxProfile()
options = firefoxOptions()
profile.set_preference("geo.prompt.testing", True)
profile.set_preference("geo.prompt.testing.allow", True)
if command_line.headless() == "y":
options.headless = True # For Headless firefox
local_driver = webdriver.Firefox(
executable_path=GeckoDriverManager().install(),
firefox_profile=profile,
options=options)
return local_driver
def launch_gecko(
gecko_path=config.DRIVER,
profile_path=config.PROFILE,
url=config.GOMAPS,
headless=False,
log_path="/dev/null",
):
""" Launch geckodriver configured for webscraping."""
# Firefox options:
options = Options()
if headless:
options.add_argument("--headless")
# Firefox profile.
if profile_path is not None:
firefox_profile = FirefoxProfile(profile_path)
else:
firefox_profile = None
# Create webdriver.
driver = webdriver.Firefox(
executable_path=gecko_path,
options=options,
firefox_profile=firefox_profile,
service_log_path=log_path,
)
# If provided, navigate to webpage.
if url is not None:
driver.get(url)
print("Launched gecko at: {}\n".format(url), file=sys.stderr)
return driver
def new_firefox_driver(self, javascript_enabled=True):
assert javascript_enabled, 'Cannot disable javascript anymore, see: https://github.com/seleniumhq/selenium/issues/635'
from selenium.webdriver import FirefoxProfile, DesiredCapabilities
# FF does not fire events when its window is not in focus.
# Native events used to fix this.
# After FF34 FF does not support native events anymore
# We're on 48.0 now on local machines, but on 31 on travis
fp = FirefoxProfile()
# fp.set_preference("focusmanager.testmode", False)
# fp.set_preference('plugins.testmode', False)
fp.set_preference('webdriver_enable_native_events', True)
fp.set_preference('webdriver.enable.native.events', True)
fp.set_preference('enable.native.events', True)
fp.native_events_enabled = True
fp.set_preference('network.http.max-connections-per-server', 1)
fp.set_preference('network.http.max-persistent-connections-per-server',
0)
fp.set_preference('network.http.spdy.enabled', False)
fp.set_preference('network.http.pipelining', True)
fp.set_preference('network.http.pipelining.maxrequests', 8)
fp.set_preference('network.http.pipelining.ssl', True)
fp.set_preference('html5.offmainthread', False)
dc = DesiredCapabilities.FIREFOX.copy()
if not javascript_enabled:
fp.set_preference('javascript.enabled', False)
dc['javascriptEnabled'] = False
wd = webdriver.Firefox(firefox_profile=fp, capabilities=dc)
self.reahl_server.install_handler(wd)
return wd
def browser(request):
"""
initialize the selenium test case
"""
global driver
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", DOWNLOAD_DIR)
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv")
capabilities = DesiredCapabilities.FIREFOX.copy()
capabilities["acceptInsecureCerts"] = True
driver = Firefox(firefox_profile=profile, capabilities=capabilities,
executable_path=os.getenv("FIREFOX_DRIVER_EXEC_PATH", "/usr/local/bin/geckodriver"))
driver.implicitly_wait(10)
driver.maximize_window()
request.addfinalizer(lambda *args: driver.quit())
yield driver
time.sleep(5)
def __enter__(self):
options = FirefoxOptions()
options.set_headless(True)
profile = FirefoxProfile()
self.browser = Firefox(firefox_options=options,
firefox_profile=profile)
return self
def driver_factory(browser, executor):
if browser == "chrome":
logger = logging.getLogger('chrome_fixture')
logger.setLevel(LOG_LEVEL)
options = ChromeOptions()
options.headless = True
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
logger.info("Подготовка среды для запуска тестов...")
options.add_experimental_option('w3c', False)
driver = EventFiringWebDriver(
webdriver.Remote(command_executor=f"http://{executor}:4444/wd/hub",
desired_capabilities={
"browserName": browser,
"platform": "WIN10",
"platformName": "WIN10"
},
options=options), MyListener())
logger.debug(
"Браузер Chrome запущен со следующими desired_capabilities:{}".
format(driver.desired_capabilities))
elif browser == "firefox":
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
options = FirefoxOptions()
options.headless = True
driver = webdriver.Firefox(options=options, firefox_profile=profile)
else:
raise Exception("Driver not supported")
return driver
def driver_initial():
"""
初始化浏览器对象并序列化
:return:
"""
client_socket = socket.socket(AF_INET, SOCK_STREAM)
try:
# setdefaulttimeout(1) 导致启动浏览器异常 设为较大时间 如 10 无异常
client_socket.settimeout(2)
client_socket.connect(('127.0.0.1', 4444))
client_socket.close()
print('4444端口已占用，geckodriver已启动')
return True
except Exception as e:
print('Error :', e)
print('4444端口未占用，geckodriver未启动')
ShellExecute(0, 'open', 'geckodriver', '', '', 1)
# ShellExecute(hwnd, op, file, params, dir, bShow)
# 其参数含义如下所示。
# hwnd：父窗口的句柄，如果没有父窗口，则为0。
# op：要进行的操作，为“open”、“print”或者为空。
# file：要运行的程序，或者打开的脚本。
# params：要向程序传递的参数，如果打开的为文件，则为空。
# dir：程序初始化的目录。
# bShow：是否显示窗口。
# firefox.exe -ProfileManager -no-remote
driver = webdriver.remote.webdriver.WebDriver(
command_executor="http://127.0.0.1:4444",
browser_profile=FirefoxProfile(FIREFOX_DIR),
desired_capabilities=DesiredCapabilities.FIREFOX)
# driver.get('about:blank')
put_browser(driver)
return False
def create_driver(self):
if 1:
caps = DesiredCapabilities().FIREFOX.copy()
profile_path = path.expanduser(
'~') + '/.mozilla/firefox/' + self.account['name']
# caps['proxy'] = {
caps['moz:firefoxOptions'] = {
"args": ["-profile", profile_path], # geckodriver 0.18+
}
profile = FirefoxProfile(profile_path)
#profile.set_preference("general.useragent.override", 'Mozilla/5.0 (X11; Linux x86_64; rv:56.0) Gecko/20100101 Firefox/56.0')
self.driver = Firefox(profile, capabilities=caps)
#self.driver = Firefox(profile)
else: # PhantomJS
# https://github.com/detro/ghostdriver
caps = DesiredCapabilities().PHANTOMJS
caps["phantomjs.page.settings.userAgent"] = \
'Mozilla/5.0 (X11; Linux x86_64; rv:56.0) Gecko/20100101 Firefox/56.0'
service_args = [
'--proxy={}'.format(':'.join(
self.account['Proxy'].split(':')[:2])),
'--proxy-type=http',
]
print(service_args)
self.driver = PhantomJS(service_args=service_args,
capabilities=caps)
self.driver.set_window_size(1120, 550)
def setOptions(self):
if self.browser == "Chrome":
# self.options.add_argument('--headless')
self.options.add_argument('--disable-gpu')
self.options.add_argument('--ignore-certificate-errors')
self.options.add_argument('--allow-running-insecure-content')
self.options.add_argument('--process-per-site')
self.options.add_argument('--disable-plugins')
self.options.add_argument('--hide-scrollbars')
self.options.add_argument('--log-level=3')
self.options.add_argument('--silent')
elif self.browser == "Firefox":
self.capabilities = DesiredCapabilities.FIREFOX.copy()
self.capabilities['platform'] = "WINDOWS"
print(self.capabilities)
self.options = Options()
profile = FirefoxProfile()
profile.set_preference(
'security.insecure_field_warning.contextual.enabled', False)
profile.set_preference('security.insecure_password.ui.enabled',
False)
profile.set_preference('dom.ipc.processCount', 1)
profile.set_preference('browser.tabs.remote.autostart.2', False)
self.options.add_argument("-headless")
self.options.add_argument("-new-tab")
# self.options.add_argument("-browser")
self.options.add_argument("-no-remote")
self.options.add_argument("-no-first-run")
self.options.add_argument("-app")
def openFF(self):
profile = FirefoxProfile()
profile.set_preference('network.http.phishy-userpass-length', 255)
profile.set_preference('network.automatic-ntlm-auth.trusted-uris',
'.sondait.com.br')
#profile.set_preference('dom.disable_beforeunload', 'false')
options = Options()
# options.set_headless(True)
# options.headless = True
# options.add_argument('-headless')
# options.add_argument("-headless") # Runs Chrome in headless mode.
# options.add_argument('-no-sandbox') # Bypass OS security model
# options.add_argument('-disable-gpu') # applicable to windows os only
# options.add_argument('start-maximized') #
# options.add_argument('disable-infobars')
# options.add_argument("-disable-extensions")
print("Headless FF initializing...")
self.driver = webdriver.Firefox(
executable_path='C:\Arris\geckodriver.exe',
options=options,
firefox_profile=profile)
if self.driver:
print("Headless FF initialized")
self.driver.implicitly_wait(10)
self.driver.delete_all_cookies()
return True
else:
print("Error FF not initialized")
return False
def driver_factory(browser):
if browser == "chrome":
logger = logging.getLogger('chrome_fixture')
logger.setLevel(LOG_LEVEL)
options = ChromeOptions()
options.headless = True
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
logger.info("Подготовка среды для запуска тестов...")
caps = DesiredCapabilities.CHROME
caps['loggingPrefs'] = {'performance': 'ALL', 'browser': 'ALL'}
options.add_experimental_option('w3c', False)
driver = EventFiringWebDriver(
webdriver.Chrome(desired_capabilities=caps, options=options),
MyListener())
logger.debug(
"Браузер Chrome запущен со следующими desired_capabilities:{}".
format(driver.desired_capabilities))
elif browser == "firefox":
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
options = FirefoxOptions()
options.headless = True
driver = webdriver.Firefox(options=options, firefox_profile=profile)
else:
raise Exception("Driver not supported")
return driver
def create_driver():
#driver = webdriver.Chrome("chromedriver.exe")
custom_profile = FirefoxProfile()
custom_profile.set_preference("dom.disable_beforeunload", True)
driver = webdriver.Firefox(firefox_profile=custom_profile, executable_path="geckodriver.exe")
driver.implicitly_wait(10)
return driver
def _setup_scraping(self) -> None:
"""
prepares the WebDriver for scraping the data by:
- setting up the WebDrive
- log in the user with the given credentials
- skipping the adding phone number dialog (should it appear)
:raise LoginError if not possible to login
"""
firefox_profile = FirefoxProfile()
firefox_profile.set_preference("browser.tabs.remote.autostart", False)
firefox_profile.set_preference("browser.tabs.remote.autostart.1",
False)
firefox_profile.set_preference("browser.tabs.remote.autostart.2",
False)
opts = Options()
opts.headless = self.headless
if opts.headless:
self.logger.info(colored("Run in headless mode.", 'blue'))
self.browser = Firefox(options=opts, firefox_profile=firefox_profile)
self._navigate_to_orders_page()
self._complete_sign_in_form()
if not self._signed_in_successful():
self.logger.error(
colored(
"Couldn't sign in. Maybe your credentials are incorrect?",
'red'))
print(
colored(
"Couldn't sign in. Maybe your credentials are incorrect?",
'red'))
self.browser.quit()
raise LoginError
self._skip_adding_phone_number()
def generate_firefox_profile(
self: "SeleniumTestability",
preferences: OptionalDictType = None,
accept_untrusted_certs: bool = False,
proxy: OptionalStrType = None,
) -> FirefoxProfile:
"""
Generates a firefox profile that sets up few required preferences for SeleniumTestability to support all necessary features.
Parameters:
- ``preferences`` - firefox profile preferences in dictionary format.
- ``accept_untrusted_certs`` should we accept untrusted/self-signed certificates.
- ``proxy`` proxy options
Note: If you opt out using this keyword, you are not able to get logs with ``Get Logs`` and Firefox.
"""
profile = FirefoxProfile()
if preferences:
for key, value in preferences.items(): # type: ignore
profile.set_preference(key, value)
profile.set_preference("devtools.console.stdout.content", True)
profile.accept_untrusted_certs = accept_untrusted_certs
if proxy:
profile.set_proxy(proxy)
profile.update_preferences()
return profile
def launch_browser(headless=False, timeout=4):
"""Launch a Firefox webdriver with disabled notifications,
allow page loading, optionally phantom mode
Args:
headless (bool): if True, launch in phantom/headless mode,
where you cannot see the browser (default=False)
timeout (int): time (s) to wait for a response (default=10)
Returns:
Webdriver (object): Firefox. With firefox profile settings
as explained above.
"""
# https://stackoverflow.com/questions/32953498/how-can-i-remove-notifications-and-alerts-from-browser-selenium-python-2-7-7
# https://stackoverflow.com/questions/26566799/how-to-wait-until-the-page-is-loaded-with-selenium-for-python
# https://stackoverflow.com/questions/5370762/how-to-hide-firefox-window-selenium-webdriver
# https://developer.mozilla.org/en-US/Firefox/Headless_mode#Selenium-in-Python
options = Options()
if headless:
options.add_argument('-headless')
logger.debug('Running in headless mode!')
fp = FirefoxProfile()
fp.set_preference("dom.webnotifications.enabled", False)
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.dir", os.getcwd())
fp.set_preference("browser.helperApps.neverAsk.saveToDisk",
"application/zip")
web_browser = Firefox(firefox_profile=fp,
executable_path='geckodriver',
options=options)
web_browser.implicitly_wait(timeout)
return web_browser
def _profile(self, arguments, extensions, proxy, user_agent):
""" Compile the capabilities of ChromeDriver inside the Container.
Args:
arguments (list):
extensions (list):
proxy (Proxy): unused.
user_agent (str):
Returns:
FirefoxProfile
"""
self.logger.debug('building browser profile')
profile = FirefoxProfile()
args = list(self.DEFAULT_ARGUMENTS)
if self.f(Flags.X_IMG):
args.append(('permissions.default.image', '2'))
if self.f(Flags.X_FLASH):
args.append(('dom.ipc.plugins.enabled.libflashplayer.so', 'false'))
for ext in extensions:
profile.add_extension(ext)
args.extend(arguments)
for arg_k, value in args:
profile.set_preference(arg_k, value)
if user_agent:
profile.set_preference('general.useragent.override', user_agent)
return profile
def get_user_page(url: str, profile_dir: str, outdir='.') -> None:
opt = FirefoxOptions()
# opt.add_argument('-headless')
p = FirefoxProfile(profile_dir)
p.set_preference('extensions.lastAppBuildId', '-1')
p.set_preference('network.proxy.socks', '')
p.set_preference('network.proxy.socks', '')
with Firefox(p, options=opt) as ff:
ff.get(url)
heights = []
while True:
count = 0
for elem in ff.find_elements_by_css_selector('.v1Nh3'):
try:
get_picture(ff, elem, outdir)
except Exception as e:
print(e)
heights.append(
ff.execute_script('return document.body.scrollHeight;'))
if len(heights) >= 3 and functools.reduce(
lambda a, b: a == b and a, heights[-3:]):
break
ff.execute_script(
'window.scrollTo(0, document.body.scrollHeight);')
time.sleep(2)
def load_localstorage(self, session_id):
sessions = json.load(open(self.sessions_file))
storage_path = sessions[str(session_id)]["session_path"]
url = sessions[str(session_id)]["web_url"]
# Setting useragent to the same one the session saved with
useragent = sessions[str(session_id)]["useragent"]
profile = FirefoxProfile()
profile.set_preference("general.useragent.override", useragent)
localStorage = pickle.load(open(storage_path, "rb"))
try:
browser = Firefox(profile)
except:
error("Couldn't open browser to view session!")
return
browser.get(url)
browser.delete_all_cookies()
browser.execute_script(
"window.localStorage.clear()") # clear the current localStorage
for key, value in localStorage.items():
browser.execute_script(
"window.localStorage.setItem(arguments[0], arguments[1]);",
key, value)
status(f"Session {session_id} loaded")
browser.refresh()
self.browsers.append(browser)
def webDriverFirefox():
profile = FirefoxProfile()
profile.set_preference('font.default.x-western', 'sans-serif')
profile.set_preference('font.name.serif.x-western', 'sans-serif')
profile.set_preference('font.size.variable.x-western', 14)
profile.set_preference('general.warnOnAboutConfig', False)
profile.set_preference('layout.css.devPixelsPerPx', '1.0')
profile.set_preference('browser.sessionstore.resume_from_crash', False)
profile.set_preference('dom.webnotifications.enabled', False)
profile.update_preferences()
if system() == "FreeBSD":
binary = FirefoxBinary('/usr/local/bin/firefox')
elif system() == "Linux":
binary = FirefoxBinary('/usr/bin/firefox')
elif system() == "Windows":
pass
caps = DesiredCapabilities.FIREFOX.copy()
caps['marionette'] = True
caps['screenResolution'] = '2560x1440'
# marionette setting is fixed in selenium 3.0 and above by default
driver = webdriver.Firefox(firefox_profile=profile,
capabilities=caps,
firefox_binary=binary)
driver.set_window_size(1470, 900)
return driver
def create(self) -> WebDriver:
pixel_density = current_app.config["WEBDRIVER_WINDOW"].get(
"pixel_density", 1)
if self._driver_type == "firefox":
driver_class = firefox.webdriver.WebDriver
options = firefox.options.Options()
profile = FirefoxProfile()
profile.set_preference("layout.css.devPixelsPerPx",
str(pixel_density))
kwargs: Dict[Any, Any] = dict(options=options,
firefox_profile=profile)
elif self._driver_type == "chrome":
driver_class = chrome.webdriver.WebDriver
options = chrome.options.Options()
options.add_argument(
f"--force-device-scale-factor={pixel_density}")
options.add_argument(
f"--window-size={self._window[0]},{self._window[1]}")
kwargs = dict(options=options)
else:
raise Exception(
f"Webdriver name ({self._driver_type}) not supported")
# Prepare args for the webdriver init
# Add additional configured options
for arg in current_app.config["WEBDRIVER_OPTION_ARGS"]:
options.add_argument(arg)
kwargs.update(current_app.config["WEBDRIVER_CONFIGURATION"])
logger.info("Init selenium driver")
return driver_class(**kwargs)
def getBrowser(self):
if platform.system() == 'Windows':
geckopath = os.path.join(os.getcwd(),"geckodriver","windows","geckodriver.exe")
elif platform.system() == 'Linux':
geckopath = os.path.join(os.getcwd(),"geckodriver","linux","geckodriver")
else:
geckopath = os.path.join(os.getcwd(),"geckodriver","mac","geckodriver")
if self.__headless:
options = FirefoxOptions()
options.headless = True
else:
options = FirefoxOptions()
options.headless = False
firefox_profile = FirefoxProfile()
firefox_profile.set_preference('permissions.default.stylesheet', 2)
firefox_profile.set_preference('permissions.default.image', 2)
firefox_profile.set_preference('permissions.default.image', 2)
firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
firefox_profile.set_preference("browser.download.folderList", 2)
firefox_profile.set_preference("browser.download.manager.showWhenStarting", False)
firefox_profile.set_preference("browser.download.dir", os.path.join(os.getcwd(),"tmp"))
firefox_profile.set_preference("browser.helperApps.neverAsk.saveToDisk", self.__mimeTypes)
firefox_profile.set_preference("plugin.disable_full_page_plugin_for_types", self.__mimeTypes)
firefox_profile.set_preference("pdfjs.disabled", True)
driver = Firefox(firefox_profile=firefox_profile,executable_path=geckopath,options=options)
return driver
def test_proxy(proxy):
if not proxy:
return
# r = get('https://www.google.com/', proxy)
ip, port, version = proxy.split(',')
print(f'Checking proxy {ip}:{port}')
options = Options()
options.headless = True
profile = FirefoxProfile()
profile.set_preference('network.proxy.type', 1)
profile.set_preference("network.proxy.socks", ip)
profile.set_preference("network.proxy.socks_port", int(port))
profile.set_preference("network.proxy.socks_version", int(version))
profile.update_preferences()
browser = Firefox(executable_path=r"./geckodriver",
firefox_profile=profile,
options=options)
try:
browser.get('https://showip.net/')
soup = BeautifulSoup(browser.page_source, 'html.parser')
return soup.select_one('#checkip').get('value').strip() == proxy.split(
',')[0].strip()
except KeyboardInterrupt:
raise KeyboardInterrupt('Abort')
except:
logging.error(traceback.format_exc())
finally:
browser.quit()
def main(cli=True, observations=2):
"""
Main code, including selenium driver initialization.
By default, outputs lines to stdout. If cli=False, the function returns a list of tuples with the resulting lines.
"""
try:
# XXX: remove this line to keep the log and continue from where you left off.
os.remove(LOG_PATH)
except OSError:
pass
profile = FirefoxProfile()
# The DesiredCapabilities approach does not work with firefox, see https://github.com/mozilla/geckodriver/issues/284
profile.set_preference("devtools.console.stdout.content", True)
options = Options()
# options.headless = True
driver = webdriver.Firefox(firefox_profile=profile, options=options)
try:
lines = collect(driver, observations)
if not cli:
return lines
for line in lines:
print(line)
return None
finally:
driver.close()
def get_firefox_driver():
profile = FirefoxProfile()
profile.set_preference('browser.cache.disk.enable', False)
profile.set_preference('browser.cache.memory.enable', False)
profile.set_preference('browser.cache.offline.enable', False)
profile.set_preference('network.cookie.cookieBehavior', 2)
if os.environ.get("DEV"):
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.socks", "127.0.0.1")
profile.set_preference("network.proxy.socks_port", 8050)
profile.set_preference("network.proxy.socks_version", 5)
profile.update_preferences()
caps = DesiredCapabilities().FIREFOX
caps["pageLoadStrategy"] = "none"
options = firefox_options()
options.headless = True
options.add_argument('start-maximized')
path = os.path.dirname(__file__)
if platform.system() == "Linux":
path = os.path.join(path, r"driver/geckodriver")
else:
path = os.path.join(path, r"driver/geckodriver_mac")
driver = webdriver.Firefox(executable_path=path,
firefox_options=options,
desired_capabilities=caps,
firefox_profile=profile)
driver.set_window_size(1920, 1080)
driver.implicitly_wait(1)
return driver
def __init__(self):
super().__init__()
welcome()
settings = load_settings()
self._upload = settings['upload'] # 上传的图片所在目录
self._download = settings['download'] # 下载的文件
self.sleep_time = settings["sleep_time"] # 下载网页源代码所等待的时间
self.separate = settings["separate"] # 是否分割开下载的数据文件和当前的图片
self.extention = settings["extention"]
self.mirror = settings["mirror"] # 是否使用镜像网站
brower = settings["brower"]
profile = settings["profile_path"]
# profile 自定义可以参考 https://blog.csdn.net/weixin_44676081/article/details/106322068
if brower == "firefox":
try:
self.driver = webdriver.Firefox(
firefox_profile=FirefoxProfile(profile),
executable_path=settings["webdriver_path"])
except Exception as e:
print(e)
else:
try:
options = webdriver.ChromeOptions()
options.add_argument("--user-data-dir=" + profile)
self.driver = webdriver.Chrome(
executable_path=settings["webdriver_path"])
except Exception as e:
print(e)
def create_download_dir_profile_for_firefox(path_to_download,
mime_types_file=None,
*extensions_files):
"""
Example use
| ${profile} | Create Download Dir Profile For Firefox | Artifacts | Resources/mimeTypes.rdf | Resources/webdriver_element_locator-2.0-fx.xpi | Resources/selenium_ide-2.9.1-fx.xpi |
| Open Browser Extension | https://support.spatialkey.com/spatialkey-sample-csv-data/ | ff_profile_dir=${profile} |
| Click Element | //a[contains(@href,'sample.csv.zip')] |
"""
path_to_download_check = validate_create_artifacts_dir(
path_to_download)
fp = FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.manager.alertOnEXEOpen", False)
fp.set_preference("browser.download.dir", path_to_download_check)
fp.set_preference("xpinstall.signatures.required", False)
fp.set_preference("browser.helperApps.alwaysAsk.force", False)
fp.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"application/msword;application/csv;text/csv;image/png;image/jpeg;application/pdf;text/html;text/plain;application/octet-stream"
)
fp.set_preference("pdfjs.disabled", True)
fp.update_preferences()
for single_extension in extensions_files:
fp.add_extension(single_extension)
if mime_types_file is not None:
from shutil import copy2
copy2(os.path.normpath(mime_types_file), fp.profile_dir)
logger.info("Firefox Profile Created in dir '" + fp.profile_dir + "'")
return fp.profile_dir
def _setup_proxy(self, browser):
"""Setup proxy for Firefox or Chrome
:type browser: str
:param browser: Browser name
:rtype: selenium.webdriver.FirefoxProfile or selenium.webdriver.ChromeOptions
:returns: Proxy settings
"""
host = "127.0.0.1"
port = "9050"
if browser == "firefox":
firefox_profile = FirefoxProfile()
# Direct = 0, Manual = 1, PAC = 2, AUTODETECT = 4, SYSTEM = 5
firefox_profile.set_preference("network.proxy.type", 1)
firefox_profile.set_preference("network.proxy.socks", host)
firefox_profile.set_preference("network.proxy.socks_port", int(port))
firefox_profile.update_preferences()
return firefox_profile
elif browser == "chrome":
proxy = f"socks5://{host}:{port}"
chrome_options = ChromeOptions()
chrome_options.add_argument(f"--proxy-server={proxy}")
return chrome_options
else:
logger.info(f"No proxy setting was done for {browser}")
