def get_driver(self):
profile = FirefoxProfile()
profile.set_preference(
"general.useragent.override",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"
)
return Firefox(profile)
def __init__(self):
# 淘宝个人页面
self.url='https://i.taobao.com/my_taobao.htm'
# 淘宝登录页面
self.login_url = "https://login.taobao.com/member/login.jhtml"
# 帐号和密码
self.username = ''
self.password = ''
self.proxy = '117.36.103.170'
self.port = '8118'
# Chrome浏览器
# chrome_options = webdriver.ChromeOptions()
# chrome_options.add_argument('--proxy-server=http://' + self.proxy + self.port)
# self.driver = webdriver.Chrome(chrome_options=chrome_options)
# # Firefox浏览器
profile = FirefoxProfile()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", self.proxy)
profile.set_preference("network.proxy.http_port", self.port)
profile.set_preference("network.proxy.share_proxy_settings", True)
self.driver = webdriver.Firefox(profile)
self.driver.implicitly_wait(5)
# Token
self._tb_token_ = ''
def setupCn(headlessBrowser):
options = Options()
options.set_headless(headless=headlessBrowser)
fp = FirefoxProfile()
fp.set_preference("webdriver.load.strategy", "unstable")
# fp.set_preference("XRE_NO_WINDOWS_CRASH_DIALOG=1")
driver = Firefox(firefox_profile=fp,
log_path=devnull,
firefox_options=options)
driver.get("http://cn.ca/")
# driver.set_window_position(1920, 0)
# sleep(30)
driver.maximize_window()
driver.implicitly_wait(100)
f = open(r"J:\LOCAL DEPARTMENT\Automation - DO NOT MOVE\CN Login.txt", 'r')
read = f.readline()
m = re.search("username: *", read)
username = read[m.end():].rstrip()
read = f.readline()
m = re.search("password: *", read)
password = read[m.end():].rstrip()
f.close()
driver.find_element_by_class_name("lbl").click()
driver.find_element_by_id("login_usernameNew").send_keys(username)
driver.find_element_by_id("login_passwordNew").send_keys(password)
driver.find_element_by_id("loginform_enterbutton").click()
return driver
def createProfile(self, saveDir):
profile = FirefoxProfile()
profile.set_preference("browser.download.folderList",2) # save to most recent selected folder
profile.set_preference("browser.download.manager.showWhenStarting",False)
profile.set_preference("browser.download.dir", saveDir)
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/vnd.ms-excel")
return profile
def start_driver(self, profile: bool, js_enabled: bool = False) -> None:
"""Starts a new local session of Firefox.
:return webdriver
"""
# Start display
if self.display:
self.vdisplay = Display(visible=0, size=(1024, 768))
self.vdisplay.start()
# Start browser
## get the Firefox profile object
firefoxProfile = None
if profile:
firefoxProfile = FirefoxProfile()
## Disable CSS
firefoxProfile.set_preference('permissions.default.stylesheet', 2)
## Disable images
firefoxProfile.set_preference('permissions.default.image', 2)
## Disable Flash
firefoxProfile.set_preference(
'dom.ipc.plugins.enabled.libflashplayer.so', 'false')
## Disable JS
if not js_enabled:
firefoxProfile.set_preference("javascript.enabled", 'false')
## Use the driver
self.driver = webdriver.Firefox(firefox_profile=firefoxProfile)
self.driver.maximize_window()
def webdriver_class():
profile = FirefoxProfile()
# Make sure Firefox WebDriver addon works, even if it could not be verified
profile.set_preference('xpinstall.signatures.required', False)
webdriver = Firefox(profile)
return webdriver
def _setProfile(self):
profile = FirefoxProfile()
profile.set_preference('permissions.default.image', 2)
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", self.proxy_ip)
profile.set_preference("network.proxy.http_port", self.proxy_port)
return profile
def __init__(self):
# self.log = ICrawlerLog('spider').save
self.IP = RedisClient().get()
self.UA = random.choice(user_agent_list)
option = webdriver.FirefoxOptions()
option.add_argument('-headless') # 启用无头
option.add_argument("--start-maximized")
option.add_argument('--no-sandbox')
option.add_argument('user-agent="{}"'.format(self.UA))
option.add_argument('--disable-gpu') # 禁用 GPU 硬件加速，防止出现bug
profile = FirefoxProfile()
# 激活手动代理配置（对应着在 profile（配置文件）中设置首选项）
profile.set_preference("network.proxy.type", 1)
# ip及其端口号配置为 http 协议代理
profile.set_preference("network.proxy.http", self.IP.split(':')[0])
profile.set_preference("network.proxy.http_port", self.IP.split(':')[-1])
# 所有协议共用一种 ip 及端口，如果单独配置，不必设置该项，因为其默认为 False
profile.set_preference("network.proxy.share_proxy_settings", True)
# 默认本地地址（localhost）不使用代理，如果有些域名在访问时不想使用代理可以使用类似下面的参数设置
# profile.set_preference("network.proxy.no_proxies_on", "localhost")
self.browser = webdriver.Firefox(options=option)
# self.browser = webdriver.Firefox(options=option, firefox_profile=profile, firefox_binary='C:\Program Files\Mozilla Firefox/firefox.exe')
self.browser.maximize_window()
# self.browser.get('http://www.ip138.com/')
self.browser.set_page_load_timeout(60)
self.browser.set_script_timeout(60) # 这两种设置都进行才有效
def disableImages(self):
fireFoxProfile = FirefoxProfile() # Nuovo profilo di FireFox #(anonimo credo, ma non importa, a meno che non voglia crearne uno, salvarlo e caricarlo da qua)
fireFoxProfile.set_preference('permissions.default.stylesheet', 2) # no CSS
fireFoxProfile.set_preference('permissions.default.image', 2) # no Image
# fireFoxProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so','false') # no Flash
return fireFoxProfile # restituico il profilo appena creato
def setup_browser():
world.browser = None
browser = getattr(settings, 'BROWSER', None)
if browser is None:
raise Exception('Please configure a browser in settings_test.py')
elif browser == 'Firefox':
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.browser = webdriver.Firefox(ff_profile)
elif browser == 'Chrome':
world.browser = webdriver.Chrome()
elif browser == "Headless":
world.browser = webdriver.PhantomJS(
desired_capabilities={'handlesAlerts': True})
cmd = "window.moveTo(0, 1); window.resizeTo(%s, %s);" % (1024, 768)
world.browser.execute_script(cmd)
world.client = client.Client()
world.using_selenium = False
# Make the browser size at least 1024x768
world.browser.execute_script("window.moveTo(0, 1); "
"window.resizeTo(1024, 768);")
# Wait implicitly for 2 seconds
world.browser.implicitly_wait(5)
# stash
world.memory = {}
def __init__(self, config: dict):
"""
Constructor
@param config the configuration to load options from
"""
# Get the logger
self.logger = logging.getLogger(config["log"]["name"])
# Set up firefox to run in headless mode to avoid graphical overhead
options = FirefoxOptions()
options.set_headless(True)
# Configure profile settings
profile = FirefoxProfile()
# Add the proxy if applicable
if config["mode"] == "tor":
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.socks", "127.0.0.1")
profile.set_preference("network.proxy.socks_port", 9050)
profile.set_preference("network.proxy.socks_remote_dns", True)
# Store configs, the profile and options
self.retries = config["firefox"]["retries"]
self.page_timeout = config["firefox"]["page_timeout"]
self.options = options
self.profile = profile
# Set driver to None for now
self.driver = None
def setup_browser():
firefox_profile = FirefoxProfile()
# Reduce bandwidth consumption and increase speed by disabling images, Flash
firefox_profile.set_preference('permissions.default.image', 2)
firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
return webdriver.Firefox(firefox_profile)
def setup_browser():
world.browser = None
browser = getattr(settings, 'BROWSER', "Headless")
if browser == 'Firefox':
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.browser = webdriver.Firefox(ff_profile)
elif browser == 'Chrome':
world.browser = webdriver.Chrome()
elif browser == "Headless":
ua = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 '
'Safari/537.36')
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = ua
dcap['handlesAlerts'] = True
world.browser = webdriver.PhantomJS(desired_capabilities=dcap)
world.client = client.Client()
world.using_selenium = False
world.browser.set_window_position(0, 0)
world.browser.set_window_size(1024, 768)
# Wait implicitly for 2 seconds
# world.browser.implicitly_wait(2)
# stash
world.memory = {}
def __init__(self):
self.firefox_profile=FirefoxProfile()
self.firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so','false')
self.brower = webdriver.Firefox(firefox_profile=self.firefox_profile,executable_path=os.path.join(os.path.abspath(os.path.dirname(__file__)),"Selenium\geckodriver.exe"))
super(MySpider, self).__init__()
#信号 当关闭时关闭浏览器
dispatcher.connect(self.spider_closed, signals.spider_closed)
def setup_browser():
world.browser = None
browser = getattr(settings, 'BROWSER', "Headless")
if browser == 'Firefox':
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.browser = webdriver.Firefox(ff_profile)
elif browser == 'Chrome':
world.browser = webdriver.Chrome()
elif browser == "Headless":
ua = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 '
'Safari/537.36')
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = ua
dcap['handlesAlerts'] = True
world.browser = webdriver.PhantomJS(desired_capabilities=dcap)
world.client = client.Client()
world.using_selenium = False
world.browser.set_window_position(0, 0)
world.browser.set_window_size(1024, 768)
# Wait implicitly for 2 seconds
# world.browser.implicitly_wait(2)
# stash
world.memory = {}
def scrape_steamer_projections():
today = datetime.today()
download_dir = os.getcwd()
default_filepath = os.path.join(download_dir, 'Fangraphs Leaderboard.csv')
desired_filepath = os.path.join(
download_dir, '{}_{}_{}_steamer.csv'.format(today.year, today.month,
today.day))
profile = FirefoxProfile()
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
'text/csv')
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", download_dir)
profile.set_preference("browser.download.folderList",
2) # download to last location set
driver = webdriver.Firefox(firefox_profile=profile)
driver.get(
"https://www.fangraphs.com/projections.aspx?pos=all&stats=bat&type=steamer"
)
driver.find_element_by_link_text('Export Data').click()
# leave browser open for 10 seconds then close
time.sleep(10)
driver.quit()
if os.path.isfile(default_filepath):
os.rename(default_filepath, desired_filepath)
print('Renamed file {} to {}'.format(default_filepath,
desired_filepath))
else:
sys.exit('Error, unable to locate file at {}'.format(default_filepath))
def click_button(links_tab):
pass
my_folder = 'C:\Users\user\Desktop\ED_pyly\WIOS\\' #'D:\userdata\lacz\Desktop\\temp\WIOS\\' # wybrac folder !!!!!
URL = "http://monitoring.krakow.pios.gov.pl/dane-pomiarowe/automatyczne/parametr/pm10/stacje/1723/dzienny/05.10.2017"
print "my folder ", my_folder
profile = FirefoxProfile()
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", my_folder)
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
'text/csv')
# profile.set_preference('network.proxy.type',2)
# profile.set_preference('network.proxy.autoconfig_url', "http://proxyconf.glb.nsn-net.net/proxy.pac")
driver = webdriver.Firefox(firefox_profile=profile)
time.sleep(5)
for link in links_tab:
driver.get(link)
time.sleep(1)
submit3 = driver.find_element_by_id("table-export-to-csv")
time.sleep(3)
submit3.click()
time.sleep(2)
assign_name(my_folder, link)
def splinterOld(url):
#start_firefox_with_firebug_plug()
#while True:
# time.sleep(1000)
#profileDir = r'C:\Users\Administrator\AppData\Roaming\Mozilla\Firefox\Profiles\82bh7hqw.default\extensions'
#profile = webdriver.FirefoxProfile(profileDir)
firefoxProfile = FirefoxProfile()
#firefoxProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'true')
firefoxProfile.set_preference('plugin.state.flash', '2')
browser = webdriver.Firefox(firefoxProfile)
#time.sleep(1)
browser.get(
"https://api.weibo.com/oauth2/authorize?client_id=3054804248&redirect_uri=http%3A%2F%2Fwww.huajiao.com&response_type=code&state=eyJzb3VyY2UiOiJzaW5hIiwicmVkaXJlY3QiOiJodHRwOlwvXC93d3cuaHVhamlhby5jb21cLz9ocmQ0Nzk1IiwidXNlcl9yYW5kIjoiNDY5YWM3NDUzNWY2MjkwMDdhOTZiYjcyNDExMGM2MjciLCJiYW5qdW1wIjoiIn0%3D"
)
#time.sleep(3)
browser.find_element_by_id("userId").clear()
browser.find_element_by_id("userId").send_keys("xiangyuzhenniao")
browser.find_element_by_id("passwd").clear()
browser.find_element_by_id('passwd').send_keys('XyznSsz1983@')
#browser.find_by_xpath('//*[@class="WB_btn_login formbtn_01"]/a').first.click()
for i in range(0, 5):
contents = browser.find_element_by_class_name("WB_btn_login")
if contents:
contents.click()
def setup_browser():
world.browser = None
browser = getattr(settings, 'BROWSER', None)
if browser is None:
raise Exception('Please configure a browser in settings_test.py')
elif browser == 'Firefox':
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.browser = webdriver.Firefox(ff_profile)
elif browser == 'Chrome':
world.browser = webdriver.Chrome()
elif browser == "Headless":
world.browser = webdriver.PhantomJS(
desired_capabilities={'handlesAlerts': True})
world.client = client.Client()
world.using_selenium = False
world.browser.set_window_position(0, 0)
world.browser.set_window_size(1024, 768)
# Wait implicitly for 2 seconds
world.browser.implicitly_wait(2)
# stash
world.memory = {}
def setup_slow_browser():
# Start the browser
profile = FirefoxProfile()
# When inserting many lines of text the Javascript process takes ages
profile.set_preference('dom.max_script_run_time', 10*60)
profile.set_preference('dom.max_chrome_script_run_time', 10*60)
world.slow_browser = webdriver.Firefox(firefox_profile=profile)
def createWebdriver():
#Set path of Driver to the one in Local Dir
gecko = os.path.normpath(
os.path.join(os.path.dirname(__file__), 'geckodriver'))
#Generate a FirefoxProfile
profile = FirefoxProfile()
#for that profile Disable- Web Push Notifications
profile.set_preference("permissions.default.desktop-notification", 1)
#Hopefully incorporate uBlock
absolutePathHack = os.environ['PWD'] #Not ideal, but she WORKS!
ublockfile = os.path.normpath(
os.path.join(absolutePathHack, '*****@*****.**'))
#Create a driver with the above settings
try:
driver = Firefox(profile, executable_path=gecko + '.exe')
except:
print("[*] Could not locate < geckodriver.exe >")
print("[*] Exiting..")
try:
#Try to add local ublock.xpi to the browser; don't crash if that fails though.
driver.install_addon(ublockfile + '.xpi', temporary=True)
except:
print(
"Could not install uBlock Origin. Is [email protected] in the same folder as the script? "
)
print(
"Try running the script from the directory it is located in, used a PWD hack for the Required AbsolutePath for AddonInstallation."
)
print(
"Will function anyway, but do you REALLY want to do this without an AdBlocker?"
)
return (driver)
def check_login_page_by_language(self, language):
if onusing_browser == "1":
firefoxProfile = FirefoxProfile()
firefoxProfile.set_preference("intl.accept_languages", language)
self.driver = webdriver.Firefox(firefox_profile=firefoxProfile)
elif onusing_browser == "2":
options = ChromeOptions()
options.add_argument("--lang=" + language)
options.add_argument("--window-size=1920,1080")
self.driver = webdriver.Chrome(chrome_options=options)
elif onusing_browser == "3":
self.assertTrue(False,
" Do not use IE, Cannot set IE browser language")
else:
print "We don't support other browsers currently."
self.driver.get(url_login)
self.driver.implicitly_wait(15)
self.check_login_page_UI_by_language(language)
time.sleep(3)
self.driver.close()
time.sleep(1)
if self.driver and onusing_browser != "1":
self.driver.quit()
def webdriver_class():
profile = FirefoxProfile()
# Make sure Firefox WebDriver addon works, even if it could not be verified
profile.set_preference('xpinstall.signatures.required', False)
webdriver = Firefox(profile)
return webdriver
def launch_browser():
if env.RUNNING_BROWSER.upper() == "FIREFOX":
# the end of the browser process , the end of the browser driven process
os.popen("TASKKILL /F /IM firefoxdriver.exe")
fp = FirefoxProfile()
fp.native_events_enabled = False
binary_path = PublicImp.common.get_value_from_conf("FIREFOX_BINARY_PATH")
if binary_path == "":
env.driver = selenium.webdriver.Firefox(firefox_profile=fp)
else:
fb = FirefoxBinary(firefox_path=binary_path)
env.driver = selenium.webdriver.Firefox(firefox_profile=fp, firefox_binary=fb)
elif env.RUNNING_BROWSER.upper() == "CHROME":
os.popen("TASKKILL /F /IM chromedriver.exe")
binary_path = PublicImp.common.get_value_from_conf("CHROME_BINARY_PATH")
chromedriver = PublicImp.common.get_value_from_conf("DRIVER_CHROME")
if binary_path == "":
os.environ["webdriver.chrome.driver"] = chromedriver
env.driver = selenium.webdriver.Chrome(executable_path=chromedriver)
else:
opts = Options()
opts.binary_location = binary_path
os.environ["webdriver.chrome.driver"] = chromedriver
env.driver = selenium.webdriver.Chrome(executable_path=chromedriver, chrome_options=opts)
elif env.RUNNING_BROWSER.upper() == "IE":
os.popen("TASKKILL /F /IM IEDriverServer.exe")
dc = DesiredCapabilities.INTERNETEXPLORER.copy()
dc['acceptSslCerts'] = True
dc['nativeEvents'] = True
iedriver = PublicImp.common.get_value_from_conf("DRIVER_IE")
os.environ["webdriver.ie.driver"] = iedriver
env.driver = selenium.webdriver.Ie(executable_path=iedriver, capabilities=dc)
else:
return False
env.platformName = env.RUNNING_BROWSER
env.TEST_URL = PublicImp.common.get_value_from_conf("TESTING_URL")
env.driver.get(env.TEST_URL)
env.driver.maximize_window()
time.sleep(3)
env.driver.refresh()
# env.driver.set_window_size(480, 800)
time.sleep(3)
return True
def setUp(self):
''' set up test '''
os.environ["DISPLAY"] = ":99"
ffx_bin = FirefoxBinary("/usr/bin/firefox")
ffx_profile = FirefoxProfile()
ffx_profile.accept_untrusted_certs = True
self.driver = webdriver.Firefox(firefox_binary=ffx_bin,
firefox_profile=ffx_profile)
def create_firefoxdriver():
options = FirefoxOptions()
profile = FirefoxProfile()
profile.set_preference("browser.cache.disk.enable", False)
profile.set_preference("browser.cache.memory.enable", False)
profile.set_preference("browser.cache.offline.enable", False)
profile.set_preference("network.cookie.cookieBehavior", 1)
return webdriver.Firefox(options=options, firefox_profile=profile)
def get_args(
driver=None,
download_dir=None,
download_ftypes=None,
firefox_pref=None,
firefox_prof_dir=None,
remote_url=None,
executable=None,
headless=False,
driver_kwargs=None,
):
"""Construct arguments to be passed to webdriver on initialization."""
kwargs = {}
firefox_profile_preferences = dict(
{
"browser.download.folderList": 2,
"browser.download.manager.showWhenStarting": False,
"browser.download.dir": download_dir,
"browser.helperApps.neverAsk.saveToDisk": download_ftypes,
"browser.helperApps.alwaysAsk.force": False,
"pdfjs.disabled": True, # disable internal ff pdf viewer to allow auto pdf download
},
**firefox_pref or {}
)
if driver == "firefox":
kwargs["profile_preferences"] = firefox_profile_preferences
kwargs["profile"] = firefox_prof_dir
elif driver == "remote":
if remote_url:
kwargs["command_executor"] = remote_url
kwargs["keep_alive"] = True
profile = FirefoxProfile(firefox_prof_dir)
for key, value in firefox_profile_preferences.items():
profile.set_preference(key, value)
kwargs["desired_capabilities"] = driver_kwargs.get("desired_capabilities", {})
kwargs["desired_capabilities"]["firefox_profile"] = profile.encoded
# remote geckodriver does not support the firefox_profile desired
# capatibility. Instead `moz:firefoxOptions` should be used:
# https://github.com/mozilla/geckodriver#firefox-capabilities
kwargs["desired_capabilities"]["moz:firefoxOptions"] = driver_kwargs.get(
"moz:firefoxOptions", {}
)
kwargs["desired_capabilities"]["moz:firefoxOptions"][
"profile"
] = profile.encoded
elif driver in ("chrome",):
if executable:
kwargs["executable_path"] = executable
if headless:
kwargs["headless"] = headless
if driver_kwargs:
kwargs.update(driver_kwargs)
return kwargs
def testCopyFromSource(self):
dir_name = tempfile.mkdtemp()
self._create_dummy_file(dir_name)
profile = FirefoxProfile()
profile.copy_profile_source(dir_name)
profile_dir = profile.path
dst_pref_file = open(os.path.join(profile_dir, self.DUMMY_FILE_NAME))
content = dst_pref_file.read()
self.assertEquals(self.DUMMY_FILE_CONTENT, content)
def get_driver():
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
profile = FirefoxProfile()
profile.add_extension(
'C:/zz_EFG/GitHub/testscript/executables/firebug-2.0.16-fx.xpi')
d = webdriver.Firefox(firefox_profile=profile)
d.implicitly_wait(7)
return d
def testCopyFromSource(self):
dir_name = tempfile.mkdtemp()
self._create_dummy_file(dir_name)
profile = FirefoxProfile()
profile.copy_profile_source(dir_name)
profile_dir = profile.path
dst_pref_file = open(os.path.join(profile_dir, self.DUMMY_FILE_NAME))
content = dst_pref_file.read()
self.assertEquals(self.DUMMY_FILE_CONTENT, content)
def setup_browser():
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.firefox = webdriver.Firefox(ff_profile)
world.client = client.Client()
world.using_selenium = False
# Make the browser size at least 1024x768
world.firefox.execute_script("window.moveTo(0, 1); window.resizeTo(1024, 768);")
def get_driver(self):
from selenium.webdriver import Firefox as FirefoxWebDriver
from selenium.webdriver import Chrome as ChromeDriver
from selenium.webdriver import Remote as RemoteDriver
# Lazilly gets the driver one time cant call in begin since ssh tunnel
# may not be created
if self._driver:
return self._driver
if self._driver_type == 'firefox':
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile # noqa
fp = FirefoxProfile()
for override in self._ff_profile_overrides:
pref, value = override.split('=')
fp.set_preference(pref, literal_eval(value))
self._profile = fp
if self._firefox_binary is None:
self._driver = FirefoxWebDriver(firefox_profile=self._profile)
else:
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary # noqa
binary = FirefoxBinary(self._firefox_binary)
self._driver = FirefoxWebDriver(
firefox_profile=self._profile,
firefox_binary=binary
)
elif self._driver_type == 'chrome':
self._driver = ChromeDriver()
else:
timeout = 60
step = 1
current = 0
while current < timeout:
try:
self._driver = RemoteDriver(
'http://%s:%s/wd/hub' % (
self._remote_server_address,
self._selenium_port,
),
self._driver_type,
'WINDOWS',
)
break
except URLError:
time.sleep(step)
current += step
except BadStatusLine:
self._driver = None
break
if current >= timeout:
raise URLError('timeout')
monkey_patch_methods(self._driver)
return self._driver
def get_args(driver=None,
download_dir=None,
download_ftypes=None,
firefox_pref=None,
firefox_prof_dir=None,
remote_url=None,
executable=None,
headless=False,
driver_kwargs=None):
"""Construct arguments to be passed to webdriver on initialization."""
kwargs = {}
firefox_profile_preferences = dict(
{
'browser.download.folderList': 2,
'browser.download.manager.showWhenStarting': False,
'browser.download.dir': download_dir,
'browser.helperApps.neverAsk.saveToDisk': download_ftypes,
'browser.helperApps.alwaysAsk.force': False,
'pdfjs.disabled':
True, # disable internal ff pdf viewer to allow auto pdf download
},
**firefox_pref or {})
if driver == 'firefox':
kwargs['profile_preferences'] = firefox_profile_preferences
kwargs['profile'] = firefox_prof_dir
if headless:
kwargs["headless"] = headless
elif driver == 'remote':
if remote_url:
kwargs['command_executor'] = remote_url
kwargs['keep_alive'] = True
profile = FirefoxProfile(firefox_prof_dir)
for key, value in firefox_profile_preferences.items():
profile.set_preference(key, value)
kwargs['desired_capabilities'] = driver_kwargs.get(
'desired_capabilities', {})
kwargs['desired_capabilities']['firefox_profile'] = profile.encoded
# remote geckodriver does not support the firefox_profile desired
# capatibility. Instead `moz:firefoxOptions` should be used:
# https://github.com/mozilla/geckodriver#firefox-capabilities
kwargs['desired_capabilities'][
'moz:firefoxOptions'] = driver_kwargs.get('moz:firefoxOptions', {})
kwargs['desired_capabilities']['moz:firefoxOptions'][
'profile'] = profile.encoded
elif driver in ('chrome', ):
if executable:
kwargs['executable_path'] = executable
if headless:
kwargs["headless"] = headless
if driver_kwargs:
kwargs.update(driver_kwargs)
return kwargs
def __init__(self):
profile = FirefoxProfile()
profile.set_preference("browser.startup.homepage_override.mstone",
"ignore")
self.wd = WebDriver(firefox_profile=profile)
#self.wd = WebDriver()
self.session = SessionHelper(self)
self.group = GroupHelper(self)
self.contact = ContactHelper(self)
def setUpClass(cls):
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
profile = FirefoxProfile()
profile.set_preference('geo.prompt.testing', True)
profile.set_preference('geo.prompt.testing.allow', True)
cls.browser = webdriver.Firefox(profile)
cls.browser.implicitly_wait(10)
super(RegistrationTests, cls).setUpClass()
def launchFb():
## get the Firefox profile object
#chromedriver = "/home/ubuntu/data/scripts/firefoxprofile/chromedriver"
#os.environ["webdriver.chrome.driver"] = chromedriver
#chrome_options = Options()
#chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:28282")
#webDriver = webdriver.Chrome(chromedriver, chrome_options=chrome_options)
if os.name!="nt":
#print os.listdir("/home/ubuntu/.mozilla/firefox")
firefoxProfile = FirefoxProfile("/home/ubuntu/.mozilla/firefox/Profiles/96zm8eem.default")
#print os.listdir("/home/ubuntu/.mozilla/firefox")
else:
firefoxProfile = FirefoxProfile("C:\Users\kubansal\AppData\Roaming\Mozilla\Firefox\Profiles\9ib58hax.default")
## Disable CSS
#firefoxProfile.set_preference('permissions.default.stylesheet', 2)
## Disable images
#firefoxProfile.set_preference('permissions.default.image', 2)
## Disable Flash
#firefoxProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so','false')
## Set the modified profile while creating the browser object
#http://kb.mozillazine.org/About:config_entries
#http://unix.stackexchange.com/questions/9107/how-can-i-run-firefox-on-linux-headlessly-i-e-without-requiring-libgtk-x11-2-0
wd = webdriver.Firefox(firefoxProfile)
#wd = webdriver.PhantomJS()
#wd.set_window_size(1120, 550)
#wd=webdriver.Firefox()
#jajafitufa123
wd.implicitly_wait(15)
wd.switch_to_default_content()
wd.maximize_window()
'''
if os.path.exists("cookies.pkl"):
cookies = pickle.load(open("cookies.pkl", "rb"))
for cookie in cookies:
print str(cookies.index(cookie))+ " of "+str(len(cookies))
wd.add_cookie(cookie)
'''
#pdb.set_trace()
wd.get("http://www.facebook.com")
#pdb.set_trace()
#wd.find_element_by_id("loginbutton").click()
#print "Logged in"
#wd.save_screenshot("xx2.png")
#wd.get("https://www.facebook.com/?sk=nf")
#time.sleep(10)
#wd.get("https://www.facebook.com/?ref=tn_tnmn")
#time.sleep(10)
print "loaded facebook page"
#wd.find_element_by_id("u_0_e").click()
#wd.find_element_by_id("u_0_e").click()
return wd
def setupEterm():
# fp = FirefoxProfile();
# fp.set_preference("webdriver.load.strategy", "unstable");
# driver = Firefox(log_path=devnull)
# driver = Firefox(firefox_profile=fp, log_path=devnull)
print("Starting Firefox in background...")
# print("Starting Firefox")
options = Options()
options.set_headless(headless=True)
# options.set_headless()
# options.set_headless(headless=False)
fp = FirefoxProfile()
fp.set_preference("webdriver.load.strategy", "unstable")
driver = Firefox(firefox_profile=fp,
log_path=devnull,
firefox_options=options)
driver.get("http:/etermsys.com/")
driver.maximize_window()
# time.sleep(10)
driver.implicitly_wait(20)
driver.switch_to.frame("sideNavBar")
try:
elem = driver.find_element_by_name("UserName")
elem.clear()
f = open(r"C:\Automation\Eterm Login.txt", 'r')
read = f.readline()
m = search("username: *", read)
read = read[m.end():]
elem.send_keys(read)
elem = driver.find_element_by_name("Password")
read = f.readline()
m = search("password: *", read)
read = read[m.end():]
elem.send_keys(read)
f.close()
elem = driver.find_element_by_name("button1")
elem.click()
except:
driver.switch_to_default_content()
driver.switch_to_frame("main")
wait = WebDriverWait(driver, 1000)
wait.until((EC.element_to_be_clickable(
(By.NAME, "ddl_terminal_select"))))
driver.switch_to_default_content()
driver.switch_to_frame("main")
select = Select(driver.find_element_by_name("ddl_terminal_select"))
select.select_by_visible_text("Seaport Intermodal")
elem = driver.find_element_by_name("submit_eTERM")
elem.click()
return driver
def get_download_profile():
profile = FirefoxProfile()
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
'application/csv')
profile.set_preference("browser.download.panel.shown", False)
profile.set_preference("browser.helperApps.neverAsk.openFile",
"text/csv,application/vnd.ms-excel")
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
"text/csv,application/vnd.ms-excel")
return profile
def __init__(self):
options = FirefoxOptions()
profile = FirefoxProfile()
user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:88.0) Gecko/20100101 Firefox/88.0"
profile.set_preference("general.useragent.override", user_agent)
options.add_argument("--headless")
self.driver = webdriver.Firefox(firefox_profile=profile,
options=options)
self.driver.get("https://www.cowin.gov.in/home")
time.sleep(3)
def _initialize(self):
self._init_browser()
self.profile = FirefoxProfile(self.profile_path)
self._init_profile()
self._init_extensions()
self._init_capabilities()
self._init_binary()
self._init_options()
self._init_webdriver()
def get_profile():
profile = FirefoxProfile()
# Disable CSS
# profile.set_preference('permissions.default.stylesheet', 2)
# Disable images
# profile.set_preference('permission.default.image', 2)
# Disable Flash
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so','false')
return profile
def firefox_from_proxy_settings(proxy_host='127.0.0.1', proxy_port=2006):
firefox_profile = FirefoxProfile()
firefox_profile.set_preference('network.proxy.type', 1)
firefox_profile.set_preference('network.proxy.socks', proxy_host)
firefox_profile.set_preference('network.proxy.socks_port', proxy_port)
firefox_profile.set_preference('network.proxy.socks_remote_dns', True)
firefox_profile.set_preference('network.prefetch-next', False)
driver = webdriver.Firefox(firefox_profile=firefox_profile)
return driver
def setup_browser(variables):
world.using_selenium = False
if skip_selenium():
world.browser = None
world.skipping = False
else:
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.firefox = webdriver.Firefox(ff_profile)
world.using_selenium = False
world.client = client.Client()
def disableImages(self):
## get the Firefox profile object
firefoxProfile = FirefoxProfile()
## Disable CSS
#firefoxProfile.set_preference('permissions.default.stylesheet', 2)
## Disable images
firefoxProfile.set_preference('permissions.default.image', 2)
## Disable Flash
firefoxProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
## Set the modified profile while creating the browser object
self.browserHandle = webdriver.Firefox(firefoxProfile)
def setup_browser(variables):
browser = getattr(settings, 'BROWSER', None)
if browser == 'Headless':
# this should really be called 'world.browser'
# but we'll fix that later
world.firefox = webdriver.PhantomJS()
else:
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.firefox = webdriver.Firefox(ff_profile)
world.client = client.Client()
world.using_selenium = False
def __init__(self):
firefox_profile = FirefoxProfile()
# we need to turn off stylesheets because otherwise Selenium couldn't find
# the the buttons in tests
firefox_profile.set_preference('permissions.default.stylesheet', 2)
super(WebDriver, self).__init__(
command_executor='http://127.0.0.1:4444/wd/hub',
browser_profile=firefox_profile,
desired_capabilities=DesiredCapabilities.FIREFOX)
self.set_page_load_timeout(60)
self.implicitly_wait(60)
self.base_url = "http://oioioi:8000"
self.delete_all_cookies()
def __init__(self):
firefox_profile = FirefoxProfile()
firefox_profile.set_preference('extensions.logging.enabled', 'false')
self._patch_subprocess()
self.driver = Firefox(firefox_profile)
self._unpatch_subprocess()
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__()
def getDriver(self):
if os.environ.get("WEBDRIVER", None) == "chrome":
os.environ['PATH'] += ":/usr/lib/chromium-browser"
logging.basicConfig(level=logging.DEBUG)
options = Options()
options.add_argument("--no-sandbox")
theDriver = webdriver.Chrome(chrome_options = options)
else:
profile_directory = os.path.join(os.path.dirname(__file__), "firefox-client-nossl-profile")
profile = FirefoxProfile(profile_directory)
profile.set_preference("security.default_personal_cert", "Select Automatically")
theDriver = webdriver.Firefox(firefox_profile=profile)
return theDriver
def __init__(self):
if config.RunInBrowser:
ffProfile = FirefoxProfile()
ffProfile.set_preference('permissions.default.image', 2)
self.driver = webdriver.Firefox(ffProfile)
else:
service_args = ['--load-images=false', '--ignore-ssl-errors=true', '--proxy-type=none']
phantomBinary = config.get_main_dir() + "\\Resources\\phantomjs.exe"
dcap = dict(webdriver.DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = (
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
self.driver = webdriver.PhantomJS(executable_path=phantomBinary, desired_capabilities=dcap,
service_args=service_args)
self.driver.implicitly_wait(10)
def setUp(self):
profile = FirefoxProfile()
# Ignore certification
profile.set_preference("webdriver_assume_untrusted_issuer", False)
profile.set_preference("webdriver_accept_untrusted_certs", True)
profile.accept_untrusted_certs = True
# Set user agent
profile.set_preference("general.useragent.override", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12A365 Safari/600.1.4")
profile.update_preferences()
self.driver = webdriver.Firefox(profile)
self.base_url = "http://m.finance.yahoo.co.jp/"
self.driver.implicitly_wait(30)
self.accept_next_alert = True
def get_args(driver=None,
download_dir=None,
download_ftypes=None,
firefox_pref=None,
firefox_prof_dir=None,
remote_url=None,
executable=None,
headless=False,
driver_kwargs=None):
"""Construct arguments to be passed to webdriver on initialization."""
kwargs = {}
firefox_profile_preferences = dict({
'browser.download.folderList': 2,
'browser.download.manager.showWhenStarting': False,
'browser.download.dir': download_dir,
'browser.helperApps.neverAsk.saveToDisk': download_ftypes,
'browser.helperApps.alwaysAsk.force': False,
'pdfjs.disabled': True, # disable internal ff pdf viewer to allow auto pdf download
}, **firefox_pref or {})
if driver == 'firefox':
kwargs['profile_preferences'] = firefox_profile_preferences
kwargs['profile'] = firefox_prof_dir
elif driver == 'remote':
if remote_url:
kwargs['url'] = remote_url
kwargs['keep_alive'] = True
profile = FirefoxProfile(firefox_prof_dir)
for key, value in firefox_profile_preferences.items():
profile.set_preference(key, value)
kwargs['firefox_profile'] = profile.encoded
# remote geckodriver does not support the firefox_profile desired
# capatibility. Instead `moz:firefoxOptions` should be used:
# https://github.com/mozilla/geckodriver#firefox-capabilities
kwargs['moz:firefoxOptions'] = driver_kwargs.get('moz:firefoxOptions', {})
kwargs['moz:firefoxOptions']['profile'] = profile.encoded
elif driver in ('chrome',):
if executable:
kwargs['executable_path'] = executable
if headless:
kwargs["headless"] = headless
if driver_kwargs:
kwargs.update(driver_kwargs)
return kwargs
def firefox_from_proxy_settings(
proxy_host='127.0.0.1',
proxy_port=2006
):
firefox_profile = FirefoxProfile()
firefox_profile.set_preference('network.proxy.type', 1)
firefox_profile.set_preference('network.proxy.socks', proxy_host)
firefox_profile.set_preference('network.proxy.socks_port', proxy_port)
firefox_profile.set_preference('network.proxy.socks_remote_dns', True)
firefox_profile.set_preference('network.prefetch-next', False)
driver = webdriver.Firefox(firefox_profile=firefox_profile)
return driver
def _load_firefox_profile():
# create a firefox profile using the template in data/firefox_profile.js.template
global firefox_profile_tmpdir
if firefox_profile_tmpdir is None:
firefox_profile_tmpdir = mkdtemp(prefix='firefox_profile_')
# Clean up tempdir at exit
atexit.register(rmtree, firefox_profile_tmpdir)
template = data_path.join('firefox_profile.js.template').read()
profile_json = Template(template).substitute(profile_dir=firefox_profile_tmpdir)
profile_dict = json.loads(profile_json)
profile = FirefoxProfile(firefox_profile_tmpdir)
for pref in profile_dict.iteritems():
profile.set_preference(*pref)
profile.update_preferences()
return profile
def visitwebsiteGUI(self, browser, address, lst_clickpoint):
try:
# get the Firefox profile object
firefoxProfile = FirefoxProfile()
# Disable CSS
firefoxProfile.set_preference('permissions.default.stylesheet', 2)
# Disable images
firefoxProfile.set_preference('permissions.default.image', 2)
# Disable Flash
firefoxProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
driver = webdriver.Firefox(firefoxProfile)
driver.get(address)
for tmpclick in lst_clickpoint:
click = tmpclick.split(',')
if click[1] == "True" :
for i in range(1,int(click[4])):
Order=randint(1,int(click[3]))
print("Pattern is : " +re.sub("###",str(Order), click[2]))
driver.find_element_by_xpath(re.sub("###",str(Order), click[2])).send_keys(Keys.CONTROL,Keys.ENTER)
self.logger.debug("Visitor (visitwebsiteGUI) - clicking on "+re.sub("###",str(Order), click[2]))
else :
#Selection = randint(0, len(lst_clickpoint)-1)
self.logger.debug("Visitor (visitwebsiteGUI) - clicking on "+click[0].replace("\n",""))
driver.find_element_by_xpath(click[0].replace("\n","")).send_keys(Keys.CONTROL,Keys.ENTER)
driver.close()
self.logger.debug("Visitor (visitwebsiteGUI) - Visit site "+address+" Successful "+";"+browser)
except Exception,e:
self.logger.error("Visitor (visitwebsiteGUI) - Visit site "+str(e))
pass
def tsetup(my_driver=""):
if platform.system() == 'Windows':
print ("Platform : windows")
a= randint(1,2)
if my_driver == "firefox":
FF_profile = FirefoxProfile()
FF_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so','false') #--Disable Flashplayer
FF_profile.set_preference('permissions.default.stylesheet', 2)
FF_profile.set_preference('permissions.default.image',2) #--Disable Image
driver = webdriver.Firefox(firefox_profile=FF_profile) #-- initiate Firefox driver using profiler
return driver
elif my_driver == "chrome":
driver = webdriver.Chrome(executable_path="../../../utils/lib/executable/chromedriver")
return driver
elif my_driver == "phantomjs":
#Latest phantomdriver update : 27/2/2015
driver = webdriver.PhantomJS(executable_path='../../../utils/lib/executable/phantomjs-1.9.8-windows/phantomjs', service_args=['--load-images=no']) #--Disable Image
return driver
elif a == 1:
driver = webdriver.Firefox()
return driver
elif a == 2:
driver = webdriver.Chrome(executable_path="../../../utils/lib/executable/chromedriver")
return driver
elif platform.system() == 'Linux':
driver = useDriver("phantomjs")
return driver
def getDriver(httpProxy = '', type='Firefox'):
if type == 'Firefox':
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': httpProxy,
'ftpProxy': httpProxy,
'sslProxy': httpProxy,
'noProxy': '' # set this value as desired
})
firefox_profile = FirefoxProfile()
#firefox_profile.add_extension("firefox_extensions/adblock_plus-2.5.1-sm+tb+an+fx.xpi")
firefox_profile.add_extension("firefox_extensions/webdriver_element_locator-1.rev312-fx.xpi")
firefox_profile.set_preference("browser.download.folderList",2)
firefox_profile.set_preference("webdriver.load.strategy", "unstable")
#driver = webdriver.Firefox(firefox_profile = firefox_profile, proxy=proxy, firefox_binary=FirefoxBinary('/usr/bin/firefox'))
#driver = webdriver.Firefox(firefox_profile = firefox_profile, proxy=proxy, firefox_binary=FirefoxBinary("/cygdrive/c/Program\ Files\ (x86)/Mozilla\ Firefox/firefox.exe"))
driver = webdriver.Firefox(firefox_profile = firefox_profile, proxy=proxy)
elif type == 'PhantomJS': # PhantomJS
service_args = [
'--proxy='+httpProxy,
'--proxy-type=http',
]
webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.User-Agent'] = 'Mozilla/5.0 (X11; Windows x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36'
driver = webdriver.PhantomJS(executable_path='windows/phantomjs.exe', service_args=service_args)
else: # Chrome
chrome_options = webdriver.ChromeOptions()
#chrome_options.add_extension('firefox_extensions/adblockplus_1_7_4.crx')
chrome_options.add_argument('--proxy-server=%s' % httpProxy)
driver = webdriver.Chrome(executable_path='windows/chromedriver.exe', chrome_options=chrome_options)
return driver
def get_profile():
profile = FirefoxProfile()
profile.set_preference('permissions.default.stylesheet', 2)
profile.set_preference('permissions.default.image', 2)
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
return profile
def getDriver():
if os.environ.get("WEBDRIVER", None) == "chrome":
os.environ['PATH'] += ":/usr/lib/chromium-browser"
options = Options()
options.add_argument("--no-sandbox")
d = DesiredCapabilities.CHROME
d['loggingPrefs'] = { 'browser':'ALL' }
theDriver = webdriver.Chrome(chrome_options = options, desired_capabilities=d)
else:
d = DesiredCapabilities.FIREFOX
d['marionette'] = True
d['loggingPrefs'] = { 'browser':'ALL' }
profile_directory = os.path.join(os.path.dirname(__file__),"..", "firefox-client-nossl-profile")
profile = FirefoxProfile(profile_directory)
profile.set_preference('webdriver.log.file', '/tmp/firefox_console')
profile.set_preference("security.default_personal_cert", "Select Automatically")
theDriver = webdriver.Firefox(firefox_profile=profile, capabilities=d)
return theDriver
def _load_firefox_profile():
# create a firefox profile using the template in data/firefox_profile.js.template
# Make a new firefox profile dir if it's unset or doesn't exist for some reason
firefox_profile_tmpdir = mkdtemp(prefix='firefox_profile_')
log.debug("created firefox profile")
# Clean up tempdir at exit
atexit.register(rmtree, firefox_profile_tmpdir, ignore_errors=True)
template = data_path.join('firefox_profile.js.template').read()
profile_json = Template(template).substitute(profile_dir=firefox_profile_tmpdir)
profile_dict = json.loads(profile_json)
profile = FirefoxProfile(firefox_profile_tmpdir)
for pref in profile_dict.items():
profile.set_preference(*pref)
profile.update_preferences()
return profile
