def __init__(self):
# self.log = ICrawlerLog('spider').save
self.UA = random.choice(useragent.agent_list)
option = webdriver.FirefoxOptions()
option.add_argument('-headless') # 启用无头
option.add_argument("--start-maximized")
option.add_argument('--no-sandbox')
option.add_argument('user-agent="{}"'.format(self.UA))
option.add_argument('--disable-gpu') # 禁用 GPU 硬件加速，防止出现bug
profile = FirefoxProfile()
# 激活手动代理配置（对应着在 profile（配置文件）中设置首选项）
profile.set_preference("network.proxy.type", 1)
# ip及其端口号配置为 http 协议代理
# profile.set_preference("network.proxy.http", self.IP.split(':')[0])
# profile.set_preference("network.proxy.http_port", self.IP.split(':')[-1])
# 所有协议共用一种 ip 及端口，如果单独配置，不必设置该项，因为其默认为 False
profile.set_preference("network.proxy.share_proxy_settings", True)
# 默认本地地址（localhost）不使用代理，如果有些域名在访问时不想使用代理可以使用类似下面的参数设置
# profile.set_preference("network.proxy.no_proxies_on", "localhost")
self.browser = webdriver.Firefox(options=option)
# self.browser = webdriver.Firefox(options=option, firefox_profile=profile, firefox_binary='C:\Program Files\Mozilla Firefox/firefox.exe')
self.browser.maximize_window()
# self.browser.get('http://www.ip138.com/')
self.browser.set_page_load_timeout(60)
self.browser.set_script_timeout(60) # 这两种设置都进行才有效
def init_firefox(request):
"""
initialise geckodriver
"""
#windowHeight=1049
#windowWidth=1790
#innerHeight=975
delta_height = 74 # MacOS
#delta_height = 74 # Ubuntu
#delta_width = 76
firefox_capabilities = DesiredCapabilities.FIREFOX.copy()
firefox_options = FirefoxOptions()
profile = FirefoxProfile()
if not check_for_test_webcam():
profile.set_preference('media.navigator.streams.fake', True)
profile.set_preference('media.navigator.permission.disabled', True)
firefox_options.profile = profile
firefox_driver = webdriver.Firefox(options=firefox_options,
capabilities=firefox_capabilities)
firefox_driver.set_window_position(0, 0)
firefox_driver.set_window_size(INNER_WIDTH, INNER_HEIGHT + delta_height)
request.cls.driver = firefox_driver
yield
firefox_driver.close()
def session_create(config):
log.info("Creating session")
options = Options()
if config.get('headless', False) is True:
log.info("Headless mode")
options.add_argument("--headless")
if config.get('webdriver_enabled') is False:
options.set_preference("dom.webdriver.enabled", False)
# selenium_profile = webdriver.FirefoxProfile('/home/container-dev/.mozilla/firefox/')
# selenium_profile.update_preferences()
# options.add_argument("-profile")
# options.add_argument("/tmp/ff1")
# driver = webdriver.Firefox(options=options, service_log_path=path.join("..", "data","geckodriver.log"))
# driver = webdriver.Chrome()
profile = FirefoxProfile()
profile.set_preference("dom.webdriver.enabled", False)
profile.set_preference('useAutomationExtension', False)
profile.update_preferences()
desired = DesiredCapabilities.FIREFOX
driver = webdriver.Firefox(options=options,
firefox_profile=profile,
desired_capabilities=desired)
log.info("New session is: %s %s" %
(driver.session_id, driver.command_executor._url))
return driver
def setup_browser():
world.browser = None
browser = getattr(settings, 'BROWSER', "Headless")
if browser == 'Firefox':
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.browser = webdriver.Firefox(ff_profile)
elif browser == 'Chrome':
world.browser = webdriver.Chrome()
elif browser == "Headless":
ua = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 '
'Safari/537.36')
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = ua
dcap['handlesAlerts'] = True
world.browser = webdriver.PhantomJS(desired_capabilities=dcap)
world.client = client.Client()
world.using_selenium = False
world.browser.set_window_position(0, 0)
world.browser.set_window_size(1024, 768)
# Wait implicitly for 2 seconds
# world.browser.implicitly_wait(2)
# stash
world.memory = {}
def setup_browser():
firefox_profile = FirefoxProfile()
# Reduce bandwidth consumption and increase speed by disabling images, Flash
firefox_profile.set_preference('permissions.default.image', 2)
firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
return webdriver.Firefox(firefox_profile)
def setup_browser():
world.browser = None
browser = getattr(settings, 'BROWSER', None)
if browser is None:
raise Exception('Please configure a browser in settings_test.py')
elif browser == 'Firefox':
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.browser = webdriver.Firefox(ff_profile)
elif browser == 'Chrome':
world.browser = webdriver.Chrome()
elif browser == "Headless":
world.browser = webdriver.PhantomJS(
desired_capabilities={'handlesAlerts': True})
cmd = "window.moveTo(0, 1); window.resizeTo(%s, %s);" % (1024, 768)
world.browser.execute_script(cmd)
world.client = client.Client()
world.using_selenium = False
# Make the browser size at least 1024x768
world.browser.execute_script("window.moveTo(0, 1); "
"window.resizeTo(1024, 768);")
# Wait implicitly for 2 seconds
world.browser.implicitly_wait(5)
# stash
world.memory = {}
def setupCn(headlessBrowser):
options = Options()
options.set_headless(headless=headlessBrowser)
fp = FirefoxProfile()
fp.set_preference("webdriver.load.strategy", "unstable")
# fp.set_preference("XRE_NO_WINDOWS_CRASH_DIALOG=1")
driver = Firefox(firefox_profile=fp,
log_path=devnull,
firefox_options=options)
driver.get("http://cn.ca/")
# driver.set_window_position(1920, 0)
# sleep(30)
driver.maximize_window()
driver.implicitly_wait(100)
f = open(r"J:\LOCAL DEPARTMENT\Automation - DO NOT MOVE\CN Login.txt", 'r')
read = f.readline()
m = re.search("username: *", read)
username = read[m.end():].rstrip()
read = f.readline()
m = re.search("password: *", read)
password = read[m.end():].rstrip()
f.close()
driver.find_element_by_class_name("lbl").click()
driver.find_element_by_id("login_usernameNew").send_keys(username)
driver.find_element_by_id("login_passwordNew").send_keys(password)
driver.find_element_by_id("loginform_enterbutton").click()
return driver
def getDriver():
try:
if os.environ.get("WEBDRIVER", None) == "chrome":
os.environ['PATH'] += ":/usr/lib/chromium-browser"
options = Options()
options.add_argument("--no-sandbox")
d = DesiredCapabilities.CHROME
d['loggingPrefs'] = {'browser': 'ALL'}
theDriver = webdriver.Chrome(chrome_options=options,
desired_capabilities=d)
else:
d = DesiredCapabilities.FIREFOX
d['marionette'] = True
d['loggingPrefs'] = {'browser': 'ALL'}
profile_directory = os.path.join(os.path.dirname(__file__), "..",
"firefox-client-nossl-profile")
profile = FirefoxProfile(profile_directory)
profile.set_preference('webdriver.log.file',
'/tmp/firefox_console')
profile.set_preference("security.default_personal_cert",
"Select Automatically")
theDriver = webdriver.Firefox(firefox_profile=profile,
capabilities=d)
except:
print("Unexpected error:", sys.exc_info()[0])
traceback.print_exc()
sys.exit(1)
print("driver is {0}".format(theDriver))
return theDriver
def createWebdriver():
#Set path of Driver to the one in Local Dir
gecko = os.path.normpath(
os.path.join(os.path.dirname(__file__), 'geckodriver'))
#Generate a FirefoxProfile
profile = FirefoxProfile()
#for that profile Disable- Web Push Notifications
profile.set_preference("permissions.default.desktop-notification", 1)
#Hopefully incorporate uBlock
absolutePathHack = os.environ['PWD'] #Not ideal, but she WORKS!
ublockfile = os.path.normpath(
os.path.join(absolutePathHack, '*****@*****.**'))
#Create a driver with the above settings
try:
driver = Firefox(profile, executable_path=gecko + '.exe')
except:
print("[*] Could not locate < geckodriver.exe >")
print("[*] Exiting..")
try:
#Try to add local ublock.xpi to the browser; don't crash if that fails though.
driver.install_addon(ublockfile + '.xpi', temporary=True)
except:
print(
"Could not install uBlock Origin. Is [email protected] in the same folder as the script? "
)
print(
"Try running the script from the directory it is located in, used a PWD hack for the Required AbsolutePath for AddonInstallation."
)
print(
"Will function anyway, but do you REALLY want to do this without an AdBlocker?"
)
return (driver)
def check_login_page_by_language(self, language):
if onusing_browser == "1":
firefoxProfile = FirefoxProfile()
firefoxProfile.set_preference("intl.accept_languages", language)
self.driver = webdriver.Firefox(firefox_profile=firefoxProfile)
elif onusing_browser == "2":
options = ChromeOptions()
options.add_argument("--lang=" + language)
options.add_argument("--window-size=1920,1080")
self.driver = webdriver.Chrome(chrome_options=options)
elif onusing_browser == "3":
self.assertTrue(False,
" Do not use IE, Cannot set IE browser language")
else:
print "We don't support other browsers currently."
self.driver.get(url_login)
self.driver.implicitly_wait(15)
self.check_login_page_UI_by_language(language)
time.sleep(3)
self.driver.close()
time.sleep(1)
if self.driver and onusing_browser != "1":
self.driver.quit()
def webdriver_class():
profile = FirefoxProfile()
# Make sure Firefox WebDriver addon works, even if it could not be verified
profile.set_preference('xpinstall.signatures.required', False)
webdriver = Firefox(profile)
return webdriver
def setup_browser():
firefox_profile = FirefoxProfile()
# Reduce bandwidth consumption and increase speed by disabling images, Flash
firefox_profile.set_preference('permissions.default.image', 2)
firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
return webdriver.Firefox(firefox_profile)
def splinterOld(url):
#start_firefox_with_firebug_plug()
#while True:
# time.sleep(1000)
#profileDir = r'C:\Users\Administrator\AppData\Roaming\Mozilla\Firefox\Profiles\82bh7hqw.default\extensions'
#profile = webdriver.FirefoxProfile(profileDir)
firefoxProfile = FirefoxProfile()
#firefoxProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'true')
firefoxProfile.set_preference('plugin.state.flash', '2')
browser = webdriver.Firefox(firefoxProfile)
#time.sleep(1)
browser.get(
"https://api.weibo.com/oauth2/authorize?client_id=3054804248&redirect_uri=http%3A%2F%2Fwww.huajiao.com&response_type=code&state=eyJzb3VyY2UiOiJzaW5hIiwicmVkaXJlY3QiOiJodHRwOlwvXC93d3cuaHVhamlhby5jb21cLz9ocmQ0Nzk1IiwidXNlcl9yYW5kIjoiNDY5YWM3NDUzNWY2MjkwMDdhOTZiYjcyNDExMGM2MjciLCJiYW5qdW1wIjoiIn0%3D"
)
#time.sleep(3)
browser.find_element_by_id("userId").clear()
browser.find_element_by_id("userId").send_keys("xiangyuzhenniao")
browser.find_element_by_id("passwd").clear()
browser.find_element_by_id('passwd').send_keys('XyznSsz1983@')
#browser.find_by_xpath('//*[@class="WB_btn_login formbtn_01"]/a').first.click()
for i in range(0, 5):
contents = browser.find_element_by_class_name("WB_btn_login")
if contents:
contents.click()
def webdriver_class():
profile = FirefoxProfile()
# Make sure Firefox WebDriver addon works, even if it could not be verified
profile.set_preference('xpinstall.signatures.required', False)
webdriver = Firefox(profile)
return webdriver
def get_driver(self):
profile = FirefoxProfile()
profile.set_preference(
"general.useragent.override",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"
)
return Firefox(profile)
def setup_slow_browser():
# Start the browser
profile = FirefoxProfile()
# When inserting many lines of text the Javascript process takes ages
profile.set_preference('dom.max_script_run_time', 10*60)
profile.set_preference('dom.max_chrome_script_run_time', 10*60)
world.slow_browser = webdriver.Firefox(firefox_profile=profile)
def getDriver(httpProxy = '', type='Firefox'):
if type == 'Firefox':
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': httpProxy,
'ftpProxy': httpProxy,
'sslProxy': httpProxy,
'noProxy': '' # set this value as desired
})
firefox_profile = FirefoxProfile()
#firefox_profile.add_extension("firefox_extensions/adblock_plus-2.5.1-sm+tb+an+fx.xpi")
firefox_profile.add_extension("firefox_extensions/webdriver_element_locator-1.rev312-fx.xpi")
firefox_profile.set_preference("browser.download.folderList",2)
firefox_profile.set_preference("webdriver.load.strategy", "unstable")
#driver = webdriver.Firefox(firefox_profile = firefox_profile, proxy=proxy, firefox_binary=FirefoxBinary('/usr/bin/firefox'))
#driver = webdriver.Firefox(firefox_profile = firefox_profile, proxy=proxy, firefox_binary=FirefoxBinary("/cygdrive/c/Program\ Files\ (x86)/Mozilla\ Firefox/firefox.exe"))
driver = webdriver.Firefox(firefox_profile = firefox_profile, proxy=proxy)
elif type == 'PhantomJS': # PhantomJS
service_args = [
'--proxy='+httpProxy,
'--proxy-type=http',
]
webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.User-Agent'] = 'Mozilla/5.0 (X11; Windows x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36'
driver = webdriver.PhantomJS(executable_path='windows/phantomjs.exe', service_args=service_args)
else: # Chrome
chrome_options = webdriver.ChromeOptions()
#chrome_options.add_extension('firefox_extensions/adblockplus_1_7_4.crx')
chrome_options.add_argument('--proxy-server=%s' % httpProxy)
driver = webdriver.Chrome(executable_path='windows/chromedriver.exe', chrome_options=chrome_options)
return driver
def init_driver():
#print (os.path.abspath(os.curdir))
ff = "geckodriver.exe"
firefox_options = Options()
firefox_options.headless = True
firefox_profile = FirefoxProfile()
firefox_profile.set_preference("permissions.default.stylesheet", 2);
firefox_profile.set_preference("permissions.default.image", 2);
#chrome_options.add_argument("--disable-gpu")
#chrome_options.add_argument("--no-sandbox")
#prefs = {"profile.managed_default_content_settings.images": 2}
#chrome_options.add_argument("headless")
#chrome_options.add_experimental_option("prefs", prefs)
try:
#driver = webdriver.firefox(executable_path=ff, options=chrome_options)
driver = webdriver.Firefox(executable_path=ff, options=firefox_options, firefox_profile = firefox_profile)
except SessionNotCreatedException:
print("Ошибка инициализации браузера. Скорее всего у вас не установлен браузер. Пожалуйста обратитесь к разработчику парсера")
driver.set_window_size(1200, 600)
return driver
def setup_browser():
world.browser = None
browser = getattr(settings, 'BROWSER', None)
if browser is None:
raise Exception('Please configure a browser in settings_test.py')
elif browser == 'Firefox':
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.browser = webdriver.Firefox(ff_profile)
elif browser == 'Chrome':
world.browser = webdriver.Chrome()
elif browser == "Headless":
world.browser = webdriver.PhantomJS(
desired_capabilities={'handlesAlerts': True})
world.client = client.Client()
world.using_selenium = False
world.browser.set_window_position(0, 0)
world.browser.set_window_size(1024, 768)
# Wait implicitly for 2 seconds
world.browser.implicitly_wait(2)
# stash
world.memory = {}
def execute(self, url):
"""
Creates a selenium webbrowser-instance( with the profile and options set in the attributes)
and retrieves Data from Website
:param url: The Url ist the Target on which the included actions will be performed on
The results will be stored in class attributes
"""
"""Creation of Webbrowser instance"""
profile = FirefoxProfile(self.profilepath)
profile.set_preference("network.cookie.cookieBehavior", 0)
profile.update_preferences()
options = Options()
options.headless = self.headless
browser = webdriver.Firefox(firefox_profile=profile,
options=options,
executable_path=self.geckopath)
browser.get(url)
"""Retrieval of Data"""
"""Get Sourcecode"""
self.source = browser.page_source
"""Get the currently used URL"""
self.trueurl = browser.current_url
"""Get Cookies from the opened website"""
self.cookies = browser.get_cookies()
"""make screenshots of the page"""
time.sleep(3)
self.imagepath = str(os.path.abspath(
os.getcwd())) + '\screenshot' + str(self)[-19:-1] + '.png'
browser.save_screenshot(self.imagepath)
"""quit Browser to clear temporary storage"""
browser.quit()
def setup_browser():
world.browser = None
browser = getattr(settings, 'BROWSER', "Headless")
if browser == 'Firefox':
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.browser = webdriver.Firefox(ff_profile)
elif browser == 'Chrome':
world.browser = webdriver.Chrome()
elif browser == "Headless":
ua = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 '
'Safari/537.36')
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = ua
dcap['handlesAlerts'] = True
world.browser = webdriver.PhantomJS(desired_capabilities=dcap)
world.client = client.Client()
world.using_selenium = False
world.browser.set_window_position(0, 0)
world.browser.set_window_size(1024, 768)
# Wait implicitly for 2 seconds
# world.browser.implicitly_wait(2)
# stash
world.memory = {}
def disableImages(self):
fireFoxProfile = FirefoxProfile() # Nuovo profilo di FireFox #(anonimo credo, ma non importa, a meno che non voglia crearne uno, salvarlo e caricarlo da qua)
fireFoxProfile.set_preference('permissions.default.stylesheet', 2) # no CSS
fireFoxProfile.set_preference('permissions.default.image', 2) # no Image
# fireFoxProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so','false') # no Flash
return fireFoxProfile # restituico il profilo appena creato
def start_browser():
firefoxProfile = FirefoxProfile()
firefoxProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'true')
firefoxProfile.set_preference("plugin.state.flash", 2)
b = webdriver.Firefox(firefoxProfile)
b.get("https://www.tanktrouble.com/")
def get_webdriver():
profile = FirefoxProfile()
profile.set_preference("browser.download.panel.shown", False)
profile.set_preference("browser.helperApps.neverAsk.openFile","text/csv,application/vnd.ms-excel")
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv,application/vnd.ms-excel")
profile.set_preference("browser.download.folderList", 2);
profile.set_preference("browser.download.dir", "c:\\firefox_downloads\\")
return webdriver.Firefox(firefox_profile=profile)
def StartBrowser(browserChoice):
if browserChoice == 1:
print '\nLaunching Chrome'
browser = webdriver.Chrome()
if browserChoice == 2:
print '\nLaunching Firefox/Iceweasel'
browser = webdriver.Firefox()
if browserChoice == 3:
print '\nLaunching Firefox/Iceweasel (light)'
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
firefoxLightProfile = FirefoxProfile()
firefoxLightProfile.set_preference('permissions.default.stylesheet', 2) # Disable CSS
firefoxLightProfile.set_preference('permissions.default.image', 2) # Disable images
firefoxLightProfile.set_preference('javascript.enabled', False) # Disable javascript
firefoxLightProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', False) # Disable Flash
firefoxLightProfile.set_preference('browser.chrome.toolbar_style', 0) # Display toolbar buttons with icons only, no text
browser = webdriver.Firefox(firefoxLightProfile)
if browserChoice == 4:
print '\nLaunching PhantomJS'
browser = webdriver.PhantomJS()
if browserChoice == 5:
print '\nLaunching PhantomJS (light)'
browser = webdriver.PhantomJS()
browser.desired_capabilities['userAgent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36'
browser.desired_capabilities['javascriptEnabled'] = False
browser.desired_capabilities['loadImages'] = False
browser.desired_capabilities['webSecurityEnabled'] = False
# Open, load and close the 'config' file
with open('config', 'r') as configFile:
config = [line.strip() for line in configFile]
configFile.close()
# Sign in
browser.get('https://linkedin.com/uas/login')
emailElement = browser.find_element_by_id('session_key-login')
emailElement.send_keys(config[0])
passElement = browser.find_element_by_id('session_password-login')
passElement.send_keys(config[1])
passElement.submit()
print 'Signing in...'
time.sleep(3)
soup = BeautifulSoup(browser.page_source)
if soup.find('div', {'class':'alert error'}):
print 'Error! Please verify your username and password.'
browser.quit()
elif browser.title == '403: Forbidden':
print 'LinkedIn is momentarily unavailable. Please wait a moment, then try again.'
browser.quit()
else:
print 'Success!\n'
LInBot(browser)
def get_args(
driver=None,
download_dir=None,
download_ftypes=None,
firefox_pref=None,
firefox_prof_dir=None,
remote_url=None,
executable=None,
headless=False,
driver_kwargs=None,
):
"""Construct arguments to be passed to webdriver on initialization."""
kwargs = {}
firefox_profile_preferences = dict(
{
"browser.download.folderList": 2,
"browser.download.manager.showWhenStarting": False,
"browser.download.dir": download_dir,
"browser.helperApps.neverAsk.saveToDisk": download_ftypes,
"browser.helperApps.alwaysAsk.force": False,
"pdfjs.disabled": True, # disable internal ff pdf viewer to allow auto pdf download
},
**firefox_pref or {}
)
if driver == "firefox":
kwargs["profile_preferences"] = firefox_profile_preferences
kwargs["profile"] = firefox_prof_dir
elif driver == "remote":
if remote_url:
kwargs["command_executor"] = remote_url
kwargs["keep_alive"] = True
profile = FirefoxProfile(firefox_prof_dir)
for key, value in firefox_profile_preferences.items():
profile.set_preference(key, value)
kwargs["desired_capabilities"] = driver_kwargs.get("desired_capabilities", {})
kwargs["desired_capabilities"]["firefox_profile"] = profile.encoded
# remote geckodriver does not support the firefox_profile desired
# capatibility. Instead `moz:firefoxOptions` should be used:
# https://github.com/mozilla/geckodriver#firefox-capabilities
kwargs["desired_capabilities"]["moz:firefoxOptions"] = driver_kwargs.get(
"moz:firefoxOptions", {}
)
kwargs["desired_capabilities"]["moz:firefoxOptions"][
"profile"
] = profile.encoded
elif driver in ("chrome",):
if executable:
kwargs["executable_path"] = executable
if headless:
kwargs["headless"] = headless
if driver_kwargs:
kwargs.update(driver_kwargs)
return kwargs
def setUpClass(cls):
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
profile = FirefoxProfile()
profile.set_preference('geo.prompt.testing', True)
profile.set_preference('geo.prompt.testing.allow', True)
cls.browser = webdriver.Firefox(profile)
cls.browser.implicitly_wait(10)
super(RegistrationTests, cls).setUpClass()
def scrape_using_selenium(self, response, new_urls):
'''
We will use selenium
'''
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium.webdriver.support.ui import WebDriverWait
profile = FirefoxProfile()
profile.set_preference("dom.max_script_run_time",600)
profile.set_preference("dom.max_chrome_script_run_time",600)
#profile.set_preference('permissions.default.image', 2) # disable images
profile.set_preference('plugin.scan.plid.all', False) # disable plugin loading crap
profile.set_preference('dom.disable_open_during_load',True) # disable popups
profile.set_preference('browser.popups.showPopupBlocker',False)
from pyvirtualdisplay import Display
display = Display(visible=0, size=(800, 600))
display.start()
driver = webdriver.Firefox(profile)
pagination = True
driver.get(response.url)
while(pagination):
try:
navs_path = driver.find_elements_by_xpath('.//*[@id="results"]/ul/li[1]/a')
for path in navs_path:
url_to_follow = ""
if "http://" not in path.get_attribute("href"):
url_to_follow = self.base_url + path.get_attribute("href")
else:
url_to_follow = path.get_attribute("href")
if not (url_to_follow in self.already_added_urls):
new_urls.append(url_to_follow)
except:
pass
finally:
if len(driver.find_elements_by_xpath('.//div[@id="results"]')) > 0:
nextbtns = driver.find_elements_by_xpath('.//div[@id="sortShowTop"]/ul[@class="show"]/li//a/div[@class="nextButton"]')
if len(nextbtns) > 0:
driver.find_element_by_xpath('.//div[@id="sortShowTop"]/ul[@class="show"]/li//a/div[@class="nextButton"]').click()
try:
WebDriverWait(driver, 3).until(lambda s: driver.find_elements_by_xpath('.//*[@id="results"]/ul/li[1]/a').is_displayed())
except:
pass
pagination = True
else:
pagination = False
else:
pagination = False
driver.close()
display.stop()
def create_firefox_driver():
capabilities = webdriver.DesiredCapabilities().FIREFOX.copy()
capabilities['acceptSslCerts'] = True
profile = FirefoxProfile()
profile.set_preference('media.navigator.streams.fake', True)
profile.set_preference('media.navigator.permission.disabled', True)
return webdriver.Firefox(firefox_profile=profile,
capabilities=capabilities)
def setup_browser():
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.firefox = webdriver.Firefox(ff_profile)
world.client = client.Client()
world.using_selenium = False
# Make the browser size at least 1024x768
world.firefox.execute_script("window.moveTo(0, 1); window.resizeTo(1024, 768);")
def get_driver(self):
from selenium.webdriver import Firefox as FirefoxWebDriver
from selenium.webdriver import Chrome as ChromeDriver
from selenium.webdriver import Remote as RemoteDriver
# Lazilly gets the driver one time cant call in begin since ssh tunnel
# may not be created
if self._driver:
return self._driver
if self._driver_type == 'firefox':
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile # noqa
fp = FirefoxProfile()
for override in self._ff_profile_overrides:
pref, value = override.split('=')
fp.set_preference(pref, literal_eval(value))
self._profile = fp
if self._firefox_binary is None:
self._driver = FirefoxWebDriver(firefox_profile=self._profile)
else:
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary # noqa
binary = FirefoxBinary(self._firefox_binary)
self._driver = FirefoxWebDriver(
firefox_profile=self._profile,
firefox_binary=binary
)
elif self._driver_type == 'chrome':
self._driver = ChromeDriver()
else:
timeout = 60
step = 1
current = 0
while current < timeout:
try:
self._driver = RemoteDriver(
'http://%s:%s/wd/hub' % (
self._remote_server_address,
self._selenium_port,
),
self._driver_type,
'WINDOWS',
)
break
except URLError:
time.sleep(step)
current += step
except BadStatusLine:
self._driver = None
break
if current >= timeout:
raise URLError('timeout')
monkey_patch_methods(self._driver)
return self._driver
def get_args(driver=None,
download_dir=None,
download_ftypes=None,
firefox_pref=None,
firefox_prof_dir=None,
remote_url=None,
executable=None,
headless=False,
driver_kwargs=None):
"""Construct arguments to be passed to webdriver on initialization."""
kwargs = {}
firefox_profile_preferences = dict(
{
'browser.download.folderList': 2,
'browser.download.manager.showWhenStarting': False,
'browser.download.dir': download_dir,
'browser.helperApps.neverAsk.saveToDisk': download_ftypes,
'browser.helperApps.alwaysAsk.force': False,
'pdfjs.disabled':
True, # disable internal ff pdf viewer to allow auto pdf download
},
**firefox_pref or {})
if driver == 'firefox':
kwargs['profile_preferences'] = firefox_profile_preferences
kwargs['profile'] = firefox_prof_dir
if headless:
kwargs["headless"] = headless
elif driver == 'remote':
if remote_url:
kwargs['command_executor'] = remote_url
kwargs['keep_alive'] = True
profile = FirefoxProfile(firefox_prof_dir)
for key, value in firefox_profile_preferences.items():
profile.set_preference(key, value)
kwargs['desired_capabilities'] = driver_kwargs.get(
'desired_capabilities', {})
kwargs['desired_capabilities']['firefox_profile'] = profile.encoded
# remote geckodriver does not support the firefox_profile desired
# capatibility. Instead `moz:firefoxOptions` should be used:
# https://github.com/mozilla/geckodriver#firefox-capabilities
kwargs['desired_capabilities'][
'moz:firefoxOptions'] = driver_kwargs.get('moz:firefoxOptions', {})
kwargs['desired_capabilities']['moz:firefoxOptions'][
'profile'] = profile.encoded
elif driver in ('chrome', ):
if executable:
kwargs['executable_path'] = executable
if headless:
kwargs["headless"] = headless
if driver_kwargs:
kwargs.update(driver_kwargs)
return kwargs
def __init__(self):
profile = FirefoxProfile()
profile.set_preference("browser.startup.homepage_override.mstone",
"ignore")
self.wd = WebDriver(firefox_profile=profile)
#self.wd = WebDriver()
self.session = SessionHelper(self)
self.group = GroupHelper(self)
self.contact = ContactHelper(self)
def setUpClass(cls):
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
profile = FirefoxProfile()
profile.set_preference('geo.prompt.testing', True)
profile.set_preference('geo.prompt.testing.allow', True)
cls.browser = webdriver.Firefox(profile)
cls.browser.implicitly_wait(10)
super(RegistrationTests, cls).setUpClass()
def setupEterm():
# fp = FirefoxProfile();
# fp.set_preference("webdriver.load.strategy", "unstable");
# driver = Firefox(log_path=devnull)
# driver = Firefox(firefox_profile=fp, log_path=devnull)
print("Starting Firefox in background...")
# print("Starting Firefox")
options = Options()
options.set_headless(headless=True)
# options.set_headless()
# options.set_headless(headless=False)
fp = FirefoxProfile()
fp.set_preference("webdriver.load.strategy", "unstable")
driver = Firefox(firefox_profile=fp,
log_path=devnull,
firefox_options=options)
driver.get("http:/etermsys.com/")
driver.maximize_window()
# time.sleep(10)
driver.implicitly_wait(20)
driver.switch_to.frame("sideNavBar")
try:
elem = driver.find_element_by_name("UserName")
elem.clear()
f = open(r"C:\Automation\Eterm Login.txt", 'r')
read = f.readline()
m = search("username: *", read)
read = read[m.end():]
elem.send_keys(read)
elem = driver.find_element_by_name("Password")
read = f.readline()
m = search("password: *", read)
read = read[m.end():]
elem.send_keys(read)
f.close()
elem = driver.find_element_by_name("button1")
elem.click()
except:
driver.switch_to_default_content()
driver.switch_to_frame("main")
wait = WebDriverWait(driver, 1000)
wait.until((EC.element_to_be_clickable(
(By.NAME, "ddl_terminal_select"))))
driver.switch_to_default_content()
driver.switch_to_frame("main")
select = Select(driver.find_element_by_name("ddl_terminal_select"))
select.select_by_visible_text("Seaport Intermodal")
elem = driver.find_element_by_name("submit_eTERM")
elem.click()
return driver
def __init__(self):
options = FirefoxOptions()
profile = FirefoxProfile()
user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:88.0) Gecko/20100101 Firefox/88.0"
profile.set_preference("general.useragent.override", user_agent)
options.add_argument("--headless")
self.driver = webdriver.Firefox(firefox_profile=profile,
options=options)
self.driver.get("https://www.cowin.gov.in/home")
time.sleep(3)
def get_profile():
profile = FirefoxProfile()
# Disable CSS
# profile.set_preference('permissions.default.stylesheet', 2)
# Disable images
# profile.set_preference('permission.default.image', 2)
# Disable Flash
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so','false')
return profile
def GetProfile(args):
args = args if args != None else {}
folder = args.get("downloadFolder", Config.DATA_DUMP_FOLDER)
print(folder)
profile = FirefoxProfile()
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.dir", folder)
profile = Browser._getDefaultPreferences(profile)
return profile
def __init__(self):
if FIREFOX_PATH:
firefox_binary = FirefoxBinary(firefox_path=FIREFOX_PATH)
else:
firefox_binary = None
firefoxProfile = FirefoxProfile()
for _ in FIREFOX_PROFILE_PREFERENCES:
firefoxProfile.set_preference(*_)
self.firefox = webdriver.Firefox(firefox_profile=firefoxProfile,
firefox_binary=firefox_binary)
self.firefox.set_page_load_timeout(FIREFOX_PAGE_LOAD_TIMEOUT)
def setup_browser(variables):
world.using_selenium = False
if skip_selenium():
world.browser = None
world.skipping = False
else:
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.firefox = webdriver.Firefox(ff_profile)
world.using_selenium = False
world.client = client.Client()
def setup_browser(variables):
world.using_selenium = False
if skip_selenium():
world.browser = None
world.skipping = False
else:
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.firefox = webdriver.Firefox(ff_profile)
world.using_selenium = False
world.client = client.Client()
def setup_browser(variables):
browser = getattr(settings, 'BROWSER', None)
if browser == 'Headless':
# this should really be called 'world.browser'
# but we'll fix that later
world.firefox = webdriver.PhantomJS()
else:
ff_profile = FirefoxProfile()
ff_profile.set_preference("webdriver_enable_native_events", False)
world.firefox = webdriver.Firefox(ff_profile)
world.client = client.Client()
world.using_selenium = False
def disableImages(self):
## get the Firefox profile object
firefoxProfile = FirefoxProfile()
## Disable CSS
#firefoxProfile.set_preference('permissions.default.stylesheet', 2)
## Disable images
firefoxProfile.set_preference('permissions.default.image', 2)
## Disable Flash
firefoxProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
## Set the modified profile while creating the browser object
self.browserHandle = webdriver.Firefox(firefoxProfile)
def setting_option(self):
firefoxProfile = FirefoxProfile()
firefoxProfile.set_preference("plugin.state.flash", 2)
options = webdriver.FirefoxOptions()
options.add_argument('-headless')
driver = webdriver.Firefox(executable_path=self.web_driver_path,
firefox_profile=firefoxProfile,
options=options)
driver.implicitly_wait(60)
self.driver = driver
def browserInitialize(report_page):
profile = FirefoxProfile()
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",\
"application/vnd.ms-excel")
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",\
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
driver = webdriver.Firefox(firefox_profile=profile)
driver.get(report_page)
print "Browser loaded"
return driver
def _setup_browser_profile(self):
fp = FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.dir", self.DOWNLOAD_DIR + '/')
fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/force-download")
if self._debug:
fp.set_preference("webdriver.log.file", "/tmp/firefox_console")
return fp
def __init__(self):
firefox_profile = FirefoxProfile()
firefox_profile.set_preference('extensions.logging.enabled', 'false')
self._patch_subprocess()
self.driver = Firefox(firefox_profile)
self._unpatch_subprocess()
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__()
def __init__(self):
firefox_profile = FirefoxProfile()
# we need to turn off stylesheets because otherwise Selenium couldn't find
# the the buttons in tests
firefox_profile.set_preference('permissions.default.stylesheet', 2)
super(WebDriver, self).__init__(
command_executor='http://127.0.0.1:4444/wd/hub',
browser_profile=firefox_profile,
desired_capabilities=DesiredCapabilities.FIREFOX)
self.set_page_load_timeout(60)
self.implicitly_wait(60)
self.base_url = "http://oioioi:8000"
self.delete_all_cookies()
def getDriver(self):
if os.environ.get("WEBDRIVER", None) == "chrome":
os.environ['PATH'] += ":/usr/lib/chromium-browser"
logging.basicConfig(level=logging.DEBUG)
options = Options()
options.add_argument("--no-sandbox")
theDriver = webdriver.Chrome(chrome_options = options)
else:
profile_directory = os.path.join(os.path.dirname(__file__), "firefox-client-nossl-profile")
profile = FirefoxProfile(profile_directory)
profile.set_preference("security.default_personal_cert", "Select Automatically")
theDriver = webdriver.Firefox(firefox_profile=profile)
return theDriver
def __init__(self, profile=None, extensions=None, user_agent=None, profile_preferences=None):
self.old_popen = subprocess.Popen
firefox_profile = FirefoxProfile(profile)
firefox_profile.set_preference('extensions.logging.enabled', False)
firefox_profile.set_preference('network.dns.disableIPv6', False)
if user_agent is not None:
firefox_profile.set_preference('general.useragent.override', user_agent)
if profile_preferences:
for key, value in profile_preferences.iteritems():
firefox_profile.set_preference(key, value)
if extensions:
for extension in extensions:
firefox_profile.add_extension(extension)
self._patch_subprocess()
self.driver = Firefox(firefox_profile)
self._unpatch_subprocess()
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__()
def __init__(self):
if config.RunInBrowser:
ffProfile = FirefoxProfile()
ffProfile.set_preference('permissions.default.image', 2)
self.driver = webdriver.Firefox(ffProfile)
else:
service_args = ['--load-images=false', '--ignore-ssl-errors=true', '--proxy-type=none']
phantomBinary = config.get_main_dir() + "\\Resources\\phantomjs.exe"
dcap = dict(webdriver.DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = (
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
self.driver = webdriver.PhantomJS(executable_path=phantomBinary, desired_capabilities=dcap,
service_args=service_args)
self.driver.implicitly_wait(10)
def __init__(self, profile=None, extensions=None, user_agent=None,
profile_preferences=None, fullscreen=False, wait_time=2):
firefox_profile = FirefoxProfile(profile)
firefox_profile.set_preference('extensions.logging.enabled', False)
firefox_profile.set_preference('network.dns.disableIPv6', False)
if user_agent is not None:
firefox_profile.set_preference(
'general.useragent.override', user_agent)
if profile_preferences:
for key, value in profile_preferences.items():
firefox_profile.set_preference(key, value)
if extensions:
for extension in extensions:
firefox_profile.add_extension(extension)
self.driver = Firefox(firefox_profile)
if fullscreen:
ActionChains(self.driver).send_keys(Keys.F11).perform()
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__(wait_time)
def __init__(self, profile=None, extensions=None, user_agent=None,
profile_preferences=None, fullscreen=False, wait_time=2,
timeout=90, capabilities=None, headless=False, **kwargs):
firefox_profile = FirefoxProfile(profile)
firefox_profile.set_preference('extensions.logging.enabled', False)
firefox_profile.set_preference('network.dns.disableIPv6', False)
firefox_capabilities = DesiredCapabilities().FIREFOX
firefox_capabilities["marionette"] = True
if 'accepted_languages' in kwargs:
firefox_profile.set_preference('intl.accept_languages', kwargs['accepted_languages'])
del kwargs['accepted_languages']
if capabilities:
for key, value in capabilities.items():
firefox_capabilities[key] = value
if user_agent is not None:
firefox_profile.set_preference(
'general.useragent.override', user_agent)
if profile_preferences:
for key, value in profile_preferences.items():
firefox_profile.set_preference(key, value)
if extensions:
for extension in extensions:
firefox_profile.add_extension(extension)
if headless:
os.environ.update({"MOZ_HEADLESS": '1'})
binary = FirefoxBinary()
binary.add_command_line_options('-headless')
kwargs['firefox_binary'] = binary
self.driver = Firefox(firefox_profile,
capabilities=firefox_capabilities,
timeout=timeout, **kwargs)
if fullscreen:
ActionChains(self.driver).send_keys(Keys.F11).perform()
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__(wait_time)
def get_args(driver=None,
download_dir=None,
download_ftypes=None,
firefox_pref=None,
firefox_prof_dir=None,
remote_url=None,
executable=None,
headless=False,
driver_kwargs=None):
"""Construct arguments to be passed to webdriver on initialization."""
kwargs = {}
firefox_profile_preferences = dict({
'browser.download.folderList': 2,
'browser.download.manager.showWhenStarting': False,
'browser.download.dir': download_dir,
'browser.helperApps.neverAsk.saveToDisk': download_ftypes,
'browser.helperApps.alwaysAsk.force': False,
'pdfjs.disabled': True, # disable internal ff pdf viewer to allow auto pdf download
}, **firefox_pref or {})
if driver == 'firefox':
kwargs['profile_preferences'] = firefox_profile_preferences
kwargs['profile'] = firefox_prof_dir
elif driver == 'remote':
if remote_url:
kwargs['url'] = remote_url
kwargs['keep_alive'] = True
profile = FirefoxProfile(firefox_prof_dir)
for key, value in firefox_profile_preferences.items():
profile.set_preference(key, value)
kwargs['firefox_profile'] = profile.encoded
# remote geckodriver does not support the firefox_profile desired
# capatibility. Instead `moz:firefoxOptions` should be used:
# https://github.com/mozilla/geckodriver#firefox-capabilities
kwargs['moz:firefoxOptions'] = driver_kwargs.get('moz:firefoxOptions', {})
kwargs['moz:firefoxOptions']['profile'] = profile.encoded
elif driver in ('chrome',):
if executable:
kwargs['executable_path'] = executable
if headless:
kwargs["headless"] = headless
if driver_kwargs:
kwargs.update(driver_kwargs)
return kwargs
def _load_firefox_profile():
# create a firefox profile using the template in data/firefox_profile.js.template
global firefox_profile_tmpdir
if firefox_profile_tmpdir is None:
firefox_profile_tmpdir = mkdtemp(prefix='firefox_profile_')
# Clean up tempdir at exit
atexit.register(rmtree, firefox_profile_tmpdir)
template = data_path.join('firefox_profile.js.template').read()
profile_json = Template(template).substitute(profile_dir=firefox_profile_tmpdir)
profile_dict = json.loads(profile_json)
profile = FirefoxProfile(firefox_profile_tmpdir)
for pref in profile_dict.iteritems():
profile.set_preference(*pref)
profile.update_preferences()
return profile
def __init__(self, profile=None, extensions=None):
self.old_popen = subprocess.Popen
firefox_profile = FirefoxProfile(profile)
firefox_profile.set_preference('extensions.logging.enabled', 'false')
if extensions:
for extension in extensions:
firefox_profile.add_extension(extension)
self._patch_subprocess()
self.driver = Firefox(firefox_profile)
self._unpatch_subprocess()
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__()
def _load_firefox_profile():
# create a firefox profile using the template in data/firefox_profile.js.template
# Make a new firefox profile dir if it's unset or doesn't exist for some reason
firefox_profile_tmpdir = mkdtemp(prefix='firefox_profile_')
log.debug("created firefox profile")
# Clean up tempdir at exit
atexit.register(rmtree, firefox_profile_tmpdir, ignore_errors=True)
template = data_path.join('firefox_profile.js.template').read()
profile_json = Template(template).substitute(profile_dir=firefox_profile_tmpdir)
profile_dict = json.loads(profile_json)
profile = FirefoxProfile(firefox_profile_tmpdir)
for pref in profile_dict.items():
profile.set_preference(*pref)
profile.update_preferences()
return profile
def getDriver():
if os.environ.get("WEBDRIVER", None) == "chrome":
os.environ['PATH'] += ":/usr/lib/chromium-browser"
options = Options()
options.add_argument("--no-sandbox")
d = DesiredCapabilities.CHROME
d['loggingPrefs'] = { 'browser':'ALL' }
theDriver = webdriver.Chrome(chrome_options = options, desired_capabilities=d)
else:
d = DesiredCapabilities.FIREFOX
d['marionette'] = True
d['loggingPrefs'] = { 'browser':'ALL' }
profile_directory = os.path.join(os.path.dirname(__file__),"..", "firefox-client-nossl-profile")
profile = FirefoxProfile(profile_directory)
profile.set_preference('webdriver.log.file', '/tmp/firefox_console')
profile.set_preference("security.default_personal_cert", "Select Automatically")
theDriver = webdriver.Firefox(firefox_profile=profile, capabilities=d)
return theDriver
def __init__(self, default_wait_time=20, profile=None, extensions=None):
self.old_popen = subprocess.Popen
firefox_profile = FirefoxProfile(profile)
firefox_profile.set_preference('extensions.logging.enabled', False)
firefox_profile.set_preference('network.dns.disableIPv6', False)
if extensions:
for extension in extensions:
firefox_profile.add_extension(extension)
self._patch_subprocess()
self.driver = Firefox(firefox_profile)
self._unpatch_subprocess()
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__(wait_time=default_wait_time)
