def get_driver(cls,selenium_hub,browser_type):
driver=None
browser_type=browser_type.lower()
download_file_content_types = "application/octet-stream,application/vnd.ms-excel,text/csv,application/zip,application/binary"
if browser_type=='ie':
opt = ie_webdriver.Options()
opt.force_create_process_api = True
opt.ensure_clean_session = True
opt.add_argument('-private')
ie_capabilities = webdriver.DesiredCapabilities.INTERNETEXPLORER.copy()
ie_capabilities.update(opt.to_capabilities())
driver = webdriver.Remote(selenium_hub, desired_capabilities=ie_capabilities)
elif browser_type=='firefox':
firefox_profile=FirefoxProfile()
# firefox_profile参数可以在火狐浏览器中访问about:config进行查看
firefox_profile.set_preference('browser.download.folderList',2) # 0是桌面;1是“我的下载”;2是自定义
firefox_profile.set_preference('browser.download.dir',ReadConfig().config.download_dir)
firefox_profile.set_preference('browser.helperApps.neverAsk.saveToDisk',download_file_content_types)
firefox_options = Firefox_Options()
if ReadConfig().config.is_firefox_headless.lower()=='true':
firefox_options.add_argument('--headless')
driver = webdriver.Remote(selenium_hub, webdriver.DesiredCapabilities.FIREFOX.copy(),browser_profile=firefox_profile,options=firefox_options)
elif browser_type=='chrome':
chrome_options=Chrome_Options()
prefs={'download.default_directory':ReadConfig().config.download_dir,'profile.default_content_settings.popups':0}
chrome_options.add_experimental_option('prefs',prefs)
if ReadConfig().config.is_chrome_headless.lower()=='true':
chrome_options.add_argument('--headless')
driver = webdriver.Remote(selenium_hub, webdriver.DesiredCapabilities.CHROME.copy(),options=chrome_options)
else:
return driver
driver.maximize_window()
driver.delete_all_cookies()
return driver
def prepareTor():
if not Settings['TOR']:
profile = FirefoxProfile()
else:
profile = (
FirefoxProfile(Settings['PROFILE_PATH']) if Settings['PROFILE_PATH']
else FirefoxProfile()
)
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', '127.0.0.1')
profile.set_preference('network.proxy.socks_port', 9050)
profile.set_preference('network.proxy.socks_remote_dns', False)
random.shuffle(Settings['USER_AGENTS'])
profile.set_preference("general.useragent.override", Settings['USER_AGENTS'][0])
profile.set_preference("intl.accept_languages", "en-US")
profile.update_preferences()
driver = webdriver.Firefox(firefox_profile = profile,
executable_path=Settings['GECKO_DRIVER'])
driver.get("http://check.torproject.org")
driver.implicitly_wait(10)
return driver
def __init__(self, cookie_path: str = None) -> None:
log.info("Creating firefox instance.")
if settings.ads_block:
firefox_profile = FirefoxProfile(
settings.firefox_profile_rich_config)
else:
firefox_profile = FirefoxProfile(settings.firefox_profile_blank)
# Disable cache to deal with overload issue when crawling lots of videos
firefox_profile.set_preference("browser.cache.disk.enable", False)
firefox_profile.set_preference("browser.cache.memory.enable", False)
firefox_profile.set_preference("browser.cache.offline.enable", False)
firefox_profile.set_preference("network.http.use-cache", False)
log.info("Current firefox profile: {}".format(str(firefox_profile)))
firefox_option = Options()
firefox_option.headless = settings.headless
firefox_option.binary = settings.firefox_binary_path
log.info("Current firefox headless status: {}, binary path: {}".format(
settings.headless, firefox_option.binary_location))
self.browser: webdriver.Firefox = \
webdriver.Firefox(firefox_profile=firefox_profile, options=firefox_option)
self.browser.maximize_window()
self.browser.delete_all_cookies()
self.__COOKIE_LOAD_RETRY: int = 3
self.__cookie_path: str = cookie_path
if cookie_path and os.path.isfile(cookie_path):
self.browser.get(settings.initial_website)
time.sleep(5)
log.info("Loading cookie from {}.".format(cookie_path))
cookie_file = None
while not cookie_file:
try:
with open(cookie_path) as f:
cookie_file: List[str] = json.load(f)
except Exception as e:
log.warning("Failed to read cookie file from {}.".format(
cookie_path))
self.__COOKIE_LOAD_RETRY -= 1
if self.__COOKIE_LOAD_RETRY <= 0:
raise
time.sleep(random.randint(1, 5))
for cookie in cookie_file:
try:
self.browser.add_cookie(cookie)
except Exception as e:
if "youtube" not in repr(cookie).lower():
log.info(
"Failed to load cookie unrelated to YouTube {}. {}"
.format(cookie, e))
else:
log.error(
"Failed to load YouTube related cookie {}. {}.".
format(cookie, e),
exc_info=True)
self.browser.get(settings.initial_website)
log.info("Open initial website: {}".format(settings.initial_website))
self.SHORT_WAIT = 5
time.sleep(self.SHORT_WAIT)
def launchFb():
## get the Firefox profile object
#chromedriver = "/home/ubuntu/data/scripts/firefoxprofile/chromedriver"
#os.environ["webdriver.chrome.driver"] = chromedriver
#chrome_options = Options()
#chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:28282")
#webDriver = webdriver.Chrome(chromedriver, chrome_options=chrome_options)
if os.name!="nt":
#print os.listdir("/home/ubuntu/.mozilla/firefox")
firefoxProfile = FirefoxProfile("/home/ubuntu/.mozilla/firefox/Profiles/96zm8eem.default")
#print os.listdir("/home/ubuntu/.mozilla/firefox")
else:
firefoxProfile = FirefoxProfile("C:\Users\kubansal\AppData\Roaming\Mozilla\Firefox\Profiles\9ib58hax.default")
## Disable CSS
#firefoxProfile.set_preference('permissions.default.stylesheet', 2)
## Disable images
#firefoxProfile.set_preference('permissions.default.image', 2)
## Disable Flash
#firefoxProfile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so','false')
## Set the modified profile while creating the browser object
#http://kb.mozillazine.org/About:config_entries
#http://unix.stackexchange.com/questions/9107/how-can-i-run-firefox-on-linux-headlessly-i-e-without-requiring-libgtk-x11-2-0
wd = webdriver.Firefox(firefoxProfile)
#wd = webdriver.PhantomJS()
#wd.set_window_size(1120, 550)
#wd=webdriver.Firefox()
#jajafitufa123
wd.implicitly_wait(15)
wd.switch_to_default_content()
wd.maximize_window()
'''
if os.path.exists("cookies.pkl"):
cookies = pickle.load(open("cookies.pkl", "rb"))
for cookie in cookies:
print str(cookies.index(cookie))+ " of "+str(len(cookies))
wd.add_cookie(cookie)
'''
#pdb.set_trace()
wd.get("http://www.facebook.com")
#pdb.set_trace()
#wd.find_element_by_id("loginbutton").click()
#print "Logged in"
#wd.save_screenshot("xx2.png")
#wd.get("https://www.facebook.com/?sk=nf")
#time.sleep(10)
#wd.get("https://www.facebook.com/?ref=tn_tnmn")
#time.sleep(10)
print "loaded facebook page"
#wd.find_element_by_id("u_0_e").click()
#wd.find_element_by_id("u_0_e").click()
return wd
def launchFb():
if os.name != "nt":
firefoxProfile = FirefoxProfile(
"/home/ubuntu/.mozilla/firefox/96zm8eem.default")
else:
firefoxProfile = FirefoxProfile(
"C:\Users\kubansal\AppData\Roaming\Mozilla\Firefox\Profiles\9ib58hax.default"
)
wd = webdriver.Firefox(firefoxProfile)
wd.implicitly_wait(15)
wd.maximize_window()
wd.get("http://www.twitter.com")
print "loaded Twitter page"
return wd
def Start(self):
try:
profile = FirefoxProfile(
'/home/juho/.mozilla/firefox/mwad0hks.default')
except FileNotFoundError:
try:
profile = FirefoxProfile(
'/home/juho/.mozilla/firefox/axli79rw.default')
except FileNotFoundError:
profile = FirefoxProfile(
'/home/juho/.mozilla/firefox/aqljl12x.default')
self.browser = webdriver.Firefox(profile)
self.browser.implicitly_wait(3)
print('Browser ready')
def do_Xenapp(self, args):
command = docopt(str(self.doc), args)
e = ExecHelper()
e.reset("Xenapp")
if command["application"] or command["desktop"]:
citrixuser = command['<username>']
citrixpass = command['<password>']
citrixurl = command['<url>']
citrixapp = " ".join(command['APPNAME'])
citrixappmod = citrixapp.replace(" ", "_0020")
#driver = webdriver.Firefox()
profpath = os.path.join(os.getenv('HOME'), ".mozilla", "firefox",
"wyse_default")
if os.path.exists(profpath):
e.Log("using existing profile")
profile = FirefoxProfile(profpath)
else:
e.Log("using a temporary profile")
profile = FirefoxProfile()
driver = webdriver.Firefox(profile)
driver.implicitly_wait(60)
driver.get(citrixurl)
#driver.find_element_by_id("skipWizardLink").click()
driver.find_element_by_id("user").clear()
driver.find_element_by_id("user").send_keys(citrixuser)
driver.find_element_by_id("password").clear()
driver.find_element_by_id("password").send_keys(citrixpass)
driver.find_element_by_css_selector("span.rightDoor").click()
if command["desktop"]:
e.Log("finding ")
driver.find_element_by_css_selector(
"#Desktops_Text > span").click()
try:
appid = "a[id*='%s']" % citrixappmod
element = driver.find_element_by_css_selector(appid)
element.click()
#e.setExecCodes(args, "Check app launch using Process alive wfica.orig","Pass")
except Exception as e:
print str(e)
e.Log(str(e))
print "Oops app not found!!", citrixapp
e.setExecCodes(args, "Could not locate app", "Fail")
#The app takes its own sweet time coming up. Should sleep time be a parameter?
print "Waiting for 90 secs before checking for window creation"
time.sleep(90)
chkcmd = "./agent/cli/bin/windowinfo.sh '%s'" % citrixapp
e.RetOutput(chkcmd)
e.Log(e.r.getOutput())
def _setProfile(self):
profile = FirefoxProfile()
profile.set_preference('permissions.default.image', 2)
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", self.proxy_ip)
profile.set_preference("network.proxy.http_port", self.proxy_port)
return profile
def test_to_capabilities(self):
opts = Options()
assert opts.to_capabilities() == {}
profile = FirefoxProfile()
opts.profile = profile
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "profile" in caps["moz:firefoxOptions"]
assert isinstance(caps["moz:firefoxOptions"]["profile"], types.StringTypes)
assert caps["moz:firefoxOptions"]["profile"] == profile.encoded
opts.add_argument("--foo")
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "args" in caps["moz:firefoxOptions"]
assert caps["moz:firefoxOptions"]["args"] == ["--foo"]
binary = FirefoxBinary()
opts.binary = binary
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "binary" in caps["moz:firefoxOptions"]
assert isinstance(caps["moz:firefoxOptions"]["binary"], types.StringTypes)
assert caps["moz:firefoxOptions"]["binary"] == binary._start_cmd
def setupCn(headlessBrowser):
options = Options()
options.set_headless(headless=headlessBrowser)
fp = FirefoxProfile()
fp.set_preference("webdriver.load.strategy", "unstable")
# fp.set_preference("XRE_NO_WINDOWS_CRASH_DIALOG=1")
driver = Firefox(firefox_profile=fp,
log_path=devnull,
firefox_options=options)
driver.get("http://cn.ca/")
# driver.set_window_position(1920, 0)
# sleep(30)
driver.maximize_window()
driver.implicitly_wait(100)
f = open(r"J:\LOCAL DEPARTMENT\Automation - DO NOT MOVE\CN Login.txt", 'r')
read = f.readline()
m = re.search("username: *", read)
username = read[m.end():].rstrip()
read = f.readline()
m = re.search("password: *", read)
password = read[m.end():].rstrip()
f.close()
driver.find_element_by_class_name("lbl").click()
driver.find_element_by_id("login_usernameNew").send_keys(username)
driver.find_element_by_id("login_passwordNew").send_keys(password)
driver.find_element_by_id("loginform_enterbutton").click()
return driver
def parse(self, response):
url = response.url
print "\n----Parse:: " + str(
self.count) + " URL: " + str(url) + " Size of response: " + str(
len(str(response.body)))
#bug fixing stops
#print str(response.body)
selenium_urls = []
new_urls = []
# for home page. StoreView for Abercrombie & Gillihicks, HomePage for Hollister
print "NAVIGATION URL " + str(url)
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
profile = FirefoxProfile()
profile.set_preference("dom.max_script_run_time", 600)
profile.set_preference("dom.max_chrome_script_run_time", 600)
profile.set_preference('permissions.default.image',
2) # disable images
profile.set_preference('plugin.scan.plid.all',
False) # disable plugin loading crap
profile.set_preference('dom.disable_open_during_load',
True) # disable popups
profile.set_preference('browser.popups.showPopupBlocker', False)
# firefoxProfile.addExtension("firebug-1.8.1.xpi")
# firefoxProfile.setPreference("extensions.firebug.currentVersion", "1.8.1")
from pyvirtualdisplay import Display
display = Display(visible=0, size=(800, 600))
display.start()
driver = webdriver.Firefox(profile)
print "Trying: " + url
try:
driver.get(url)
pins_old = driver.find_elements_by_xpath('//div[@class="pin"]')
num_pins_old = len(pins_old)
print 'num_pins_old %d ' % num_pins
while True:
driver.execute_script("scrollTo(0, 300000)")
pins_new = driver.find_elements_by_xpath('//div[@class="pin"]')
num_pins_new = len(pins_new)
print 'num_pins_new %d num_pins_old %d' % (num_pins_new,
num_pins_old)
if num_pins_new == num_pins_old:
break
except:
pass
driver.quit()
display.stop()
display = Display(visible=0, size=(800, 600))
display.start()
driver = webdriver.Firefox(profile)
driver.quit()
display.stop()
def get_profile():
profile = FirefoxProfile()
profile.set_preference('permissions.default.stylesheet', 2)
profile.set_preference('permissions.default.image', 2)
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
return profile
def init_driver():
#print (os.path.abspath(os.curdir))
ff = "geckodriver.exe"
firefox_options = Options()
firefox_options.headless = True
firefox_profile = FirefoxProfile()
firefox_profile.set_preference("permissions.default.stylesheet", 2);
firefox_profile.set_preference("permissions.default.image", 2);
#chrome_options.add_argument("--disable-gpu")
#chrome_options.add_argument("--no-sandbox")
#prefs = {"profile.managed_default_content_settings.images": 2}
#chrome_options.add_argument("headless")
#chrome_options.add_experimental_option("prefs", prefs)
try:
#driver = webdriver.firefox(executable_path=ff, options=chrome_options)
driver = webdriver.Firefox(executable_path=ff, options=firefox_options, firefox_profile = firefox_profile)
except SessionNotCreatedException:
print("Ошибка инициализации браузера. Скорее всего у вас не установлен браузер. Пожалуйста обратитесь к разработчику парсера")
driver.set_window_size(1200, 600)
return driver
def __init__(self, timeout=20, config_dir=None):
importlib.reload(sys)
#sys.setdefaultencoding("utf8")
self.logger = add_logger("browser")
if "Linux" in platform.uname():
self.logger.info("starting headless mode")
self.display = Display(visible=0, size=(1680, 1050))
self.display.start()
else:
self.logger.info(
"MacOS or Windows detected, can't start headless mode")
# Disable images to speed up loading times
firefox_profile = FirefoxProfile()
#firefox_profile.set_preference("permissions.default.image", 2)
#firefox_profile.set_preference("permissions.default.stylesheet", 2)
self.browser = webdriver.Firefox(firefox_profile)
self.logged_in = False
self.timeout = timeout
self.config_dir = config_dir
def click_button(links_tab):
pass
my_folder = 'C:\Users\user\Desktop\ED_pyly\WIOS\\' #'D:\userdata\lacz\Desktop\\temp\WIOS\\' # wybrac folder !!!!!
URL = "http://monitoring.krakow.pios.gov.pl/dane-pomiarowe/automatyczne/parametr/pm10/stacje/1723/dzienny/05.10.2017"
print "my folder ", my_folder
profile = FirefoxProfile()
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", my_folder)
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
'text/csv')
# profile.set_preference('network.proxy.type',2)
# profile.set_preference('network.proxy.autoconfig_url', "http://proxyconf.glb.nsn-net.net/proxy.pac")
driver = webdriver.Firefox(firefox_profile=profile)
time.sleep(5)
for link in links_tab:
driver.get(link)
time.sleep(1)
submit3 = driver.find_element_by_id("table-export-to-csv")
time.sleep(3)
submit3.click()
time.sleep(2)
assign_name(my_folder, link)
def getProfile(pool):
prefs = FirefoxProfile()
random.shuffle(pool)
proxy = pool.pop()
print('Server: {0}\nPort: {1}'.format(proxy.address, proxy.port))
# Write proxy info to log file
with open('WatsonResponse.txt', 'a') as logFile:
time = dt.now().strftime('%b %d, %Y @ %H:%M:%S')
logFile.write('--------------Pass at {}--------------\n'.format(time))
logFile.write('Address: {0}\nPort: {1}\n'.format(
proxy.address, proxy.port))
prefs.set_preference('network.proxy.type', 1)
prefs.set_preference('network.proxy.share_proxy_settings', True)
prefs.set_preference('network.http.use-cache', False)
prefs.set_preference('network.proxy.http', proxy.address)
prefs.set_preference('network.proxy.http_port', proxy.port)
prefs.set_preference('network.proxy.ssl', proxy.address)
prefs.set_preference('network.proxy.ssl_port', proxy.port)
prefs.set_preference('network.proxy.socks', proxy.address)
prefs.set_preference('network.proxy.socks_port', proxy.port)
return prefs
def __init__(self):
self.firefox_profile=FirefoxProfile()
self.firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so','false')
self.brower = webdriver.Firefox(firefox_profile=self.firefox_profile,executable_path=os.path.join(os.path.abspath(os.path.dirname(__file__)),"Selenium\geckodriver.exe"))
super(MySpider, self).__init__()
#信号 当关闭时关闭浏览器
dispatcher.connect(self.spider_closed, signals.spider_closed)
def __init__(self, config: dict):
"""
Constructor
@param config the configuration to load options from
"""
# Get the logger
self.logger = logging.getLogger(config["log"]["name"])
# Set up firefox to run in headless mode to avoid graphical overhead
options = FirefoxOptions()
options.set_headless(True)
# Configure profile settings
profile = FirefoxProfile()
# Add the proxy if applicable
if config["mode"] == "tor":
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.socks", "127.0.0.1")
profile.set_preference("network.proxy.socks_port", 9050)
profile.set_preference("network.proxy.socks_remote_dns", True)
# Store configs, the profile and options
self.retries = config["firefox"]["retries"]
self.page_timeout = config["firefox"]["page_timeout"]
self.options = options
self.profile = profile
# Set driver to None for now
self.driver = None
def init_glob_vars():
global PROFILE_glob
global CAPABILITIES_glob
global SCRIPT_DIR_glob
global EXECUTABLE_PATH_glob
global PROXY_ADDRESS_glob
global PROXY_glob
# Browser settings
PROFILE_glob = FirefoxProfile(
'/Users/altay.amanbay/Library/Application Support/Firefox/Profiles')
PROFILE_glob.set_preference("network.proxy.type", 1)
PROFILE_glob.set_preference("network.proxy.socks", "127.0.0.1")
PROFILE_glob.set_preference("network.proxy.socks_port", 9150)
PROFILE_glob.set_preference("network.proxy.socks_version", 5)
PROFILE_glob.set_preference("network.proxy.socks_remote_dns", True)
PROFILE_glob.update_preferences()
CAPABILITIES_glob = DesiredCapabilities.FIREFOX
CAPABILITIES_glob["marionette"] = True
CAPABILITIES_glob[
"binary"] = "/Applications/Firefox.app/Contents/MacOS/firefox-bin"
# Proxy (start Tor browser before executing script)
PROXY_ADDRESS_glob = "127.0.0.1:9150" # localhost and Tor browser's default port number
PROXY_glob = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': PROXY_ADDRESS_glob,
})
# Webdriver path
SCRIPT_DIR_glob = os.path.dirname(os.path.abspath(__file__))
print('Abs path:', SCRIPT_DIR_glob)
EXECUTABLE_PATH_glob = SCRIPT_DIR_glob + '/webdrivers/geckodriver - v0.18.0/geckodriver'
def __init__(self,
profile=None,
extensions=None,
user_agent=None,
profile_preferences=None):
self.old_popen = subprocess.Popen
firefox_profile = FirefoxProfile(profile)
firefox_profile.set_preference('extensions.logging.enabled', False)
firefox_profile.set_preference('network.dns.disableIPv6', False)
if user_agent is not None:
firefox_profile.set_preference('general.useragent.override',
user_agent)
if profile_preferences:
for key, value in profile_preferences.iteritems():
firefox_profile.set_preference(key, value)
if extensions:
for extension in extensions:
firefox_profile.add_extension(extension)
self._patch_subprocess()
self.driver = Firefox(firefox_profile)
self._unpatch_subprocess()
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__()
def test_to_capabilities(self):
opts = Options()
firefox_caps = DesiredCapabilities.FIREFOX.copy()
firefox_caps.update({"pageLoadStrategy": "normal"})
assert opts.to_capabilities() == firefox_caps
profile = FirefoxProfile()
opts.profile = profile
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "profile" in caps["moz:firefoxOptions"]
assert isinstance(caps["moz:firefoxOptions"]["profile"], str)
assert caps["moz:firefoxOptions"]["profile"] == profile.encoded
opts.add_argument("--foo")
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "args" in caps["moz:firefoxOptions"]
assert caps["moz:firefoxOptions"]["args"] == ["--foo"]
binary = FirefoxBinary()
opts.binary = binary
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "binary" in caps["moz:firefoxOptions"]
assert isinstance(caps["moz:firefoxOptions"]["binary"], str)
assert caps["moz:firefoxOptions"]["binary"] == binary._start_cmd
opts.set_preference("spam", "ham")
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "prefs" in caps["moz:firefoxOptions"]
assert isinstance(caps["moz:firefoxOptions"]["prefs"], dict)
assert caps["moz:firefoxOptions"]["prefs"]["spam"] == "ham"
def __init__(self):
pf = {
'browser.helperApps.neverAsk.saveToDisk':
'application/x-netcdf, application/netcdf',
'browser.download.manager.closeWhenDone': True,
'browser.download.manager.showWhenStarting': False,
'browser.download.folderList': 2,
'browser.download.dir': AbstractBrowserBasedTest.DOWNLOAD_DIR_PATH
}
AbstractBrowserBasedTest._firefox_profile = FirefoxProfile() # profile
AbstractBrowserBasedTest._firefox_profile.set_preference(
'extensions.logging.enabled', False)
AbstractBrowserBasedTest._firefox_profile.set_preference(
'network.dns.disableIPv6', False)
for key, value in list(pf.items()):
AbstractBrowserBasedTest._firefox_profile.set_preference(
key, value)
AbstractBrowserBasedTest._firefox_capabilities = DesiredCapabilities(
).FIREFOX
AbstractBrowserBasedTest._firefox_capabilities['marionette'] = True
AbstractBrowserBasedTest._firefox_capabilities[
'moz:firefoxOptions'] = {
'args': [self.__get_arg()]
}
urllib3.disable_warnings()
def init(self,config):
self.config=config
print self.config.browser_driver_command_executor
command_executor=self.config.browser_driver_command_executor
if self.config.browser_profile_dir:
profile=FirefoxProfile(profile_directory=self.config.browser_profile_dir)
self.dmesg('loading browser profile')
else:
profile=None
self.dmesg('browser profile not configured, use None profile')
self.dmesg('trying start a browser')
self.driver=webdriver.Remote(
command_executor=self.config.browser_driver_command_executor,
desired_capabilities=DesiredCapabilities.FIREFOX,
browser_profile=profile)
try:
self.driver.get(self.config.start_url)
except:
self.dmesg('打开失败，10s 后重试...')
sleep(10)
try:
self.driver.get(self.config.start_url)
except:
self.dmesg('打开失败，10s 后重试...')
sleep(10)
self.driver.get(self.config.start_url)
def main():
mime = "application/pdf,text/html,text/txt,application/msword"
profile = FirefoxProfile()
profile.set_preference('browser.download.folderList', 2) # custom location
profile.set_preference('browser.download.manager.showWhenStarting', False)
profile.set_preference("browser.download.dir", '/Users/thiagovieira/Downloads')
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", mime)
profile.set_preference("pdfjs.disabled", True)
profile.set_preference("plugin.disable_full_page_plugin_for_types", mime)
driver = webdriver.Firefox(firefox_profile=profile)
driver.get("https://projudi.tjba.jus.br")
time.sleep(20)
url = 'https://projudi.tjba.jus.br/projudi/listagens/DownloadArquivo?arquivo={}'
for n in range(1, 10001):
try:
#checkinternet()
driver.set_page_load_timeout(1)
driver.get(url.format(n))
time.sleep(5)
except:
pass
driver.quit()
def create_fast_firefox_profile():
quickjava_url = 'https://addons.mozilla.org/firefox/downloads/latest/1237/addon-1237-latest.xpi'
if not os.path.isfile(get_file_name_from_url(quickjava_url)):
# download the extension
downloadFile(quickjava_url)
## get the Firefox profile object
firefox_profile = FirefoxProfile()
## Disable CSS
firefox_profile.set_preference('permissions.default.stylesheet', 2)
## Disable images
firefox_profile.set_preference('permissions.default.image', 2)
## Disable Flash
firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
firefox_profile.add_extension(get_file_name_from_url(quickjava_url))
firefox_profile.set_preference(
"thatoneguydotnet.QuickJava.curVersion",
"2.0.6.1") ## Prevents loading the 'thank you for installing screen'
firefox_profile.set_preference(
"thatoneguydotnet.QuickJava.startupStatus.Images",
2) ## Turns images off
firefox_profile.set_preference(
"thatoneguydotnet.QuickJava.startupStatus.AnimatedImage",
2) ## Turns animated images off
def make_headless_browser(custom_options={}):
"""无头浏览器"""
# 使用系统代理
proxy = Proxy()
proxy.proxy_type = 'SYSTEM'
fp = FirefoxProfile()
options = Options()
# 无头浏览器
options.headless = True
# 禁用gpu加速
options.add_argument('--disable-gpu')
# 网页加载模式
# options.page_load_strategy = 'eager'
default_options = {}
default_options.update(custom_options)
log_path = data_root('geckordriver') / f'{os.getpid()}.log'
return webdriver.Firefox(
options=options,
seleniumwire_options=default_options,
proxy=proxy,
firefox_profile=fp,
service_log_path=log_path,
executable_path=DEFAULT_CONFIG['geckodriver_path'])
def test_to_capabilities(self):
opts = Options()
assert opts.to_capabilities() == {}
profile = FirefoxProfile()
opts.profile = profile
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "profile" in caps["moz:firefoxOptions"]
assert isinstance(caps["moz:firefoxOptions"]["profile"], basestring)
assert caps["moz:firefoxOptions"]["profile"] == profile.encoded
opts.add_argument("--foo")
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "args" in caps["moz:firefoxOptions"]
assert caps["moz:firefoxOptions"]["args"] == ["--foo"]
binary = FirefoxBinary()
opts.binary = binary
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "binary" in caps["moz:firefoxOptions"]
assert isinstance(caps["moz:firefoxOptions"]["binary"], basestring)
assert caps["moz:firefoxOptions"]["binary"] == binary._start_cmd
opts.set_preference("spam", "ham")
caps = opts.to_capabilities()
assert "moz:firefoxOptions" in caps
assert "prefs" in caps["moz:firefoxOptions"]
assert isinstance(caps["moz:firefoxOptions"]["prefs"], dict)
assert caps["moz:firefoxOptions"]["prefs"]["spam"] == "ham"
def check_login_page_by_language(self, language):
if onusing_browser == "1":
firefoxProfile = FirefoxProfile()
firefoxProfile.set_preference("intl.accept_languages", language)
self.driver = webdriver.Firefox(firefox_profile=firefoxProfile)
elif onusing_browser == "2":
options = ChromeOptions()
options.add_argument("--lang=" + language)
options.add_argument("--window-size=1920,1080")
self.driver = webdriver.Chrome(chrome_options=options)
elif onusing_browser == "3":
self.assertTrue(False,
" Do not use IE, Cannot set IE browser language")
else:
print "We don't support other browsers currently."
self.driver.get(url_login)
self.driver.implicitly_wait(15)
self.check_login_page_UI_by_language(language)
time.sleep(3)
self.driver.close()
time.sleep(1)
if self.driver and onusing_browser != "1":
self.driver.quit()
def session_create(config):
log.info("Creating session")
options = Options()
if config.get('headless', False) is True:
log.info("Headless mode")
options.add_argument("--headless")
if config.get('webdriver_enabled') is False:
options.set_preference("dom.webdriver.enabled", False)
# selenium_profile = webdriver.FirefoxProfile('/home/container-dev/.mozilla/firefox/')
# selenium_profile.update_preferences()
# options.add_argument("-profile")
# options.add_argument("/tmp/ff1")
# driver = webdriver.Firefox(options=options, service_log_path=path.join("..", "data","geckodriver.log"))
# driver = webdriver.Chrome()
profile = FirefoxProfile()
profile.set_preference("dom.webdriver.enabled", False)
profile.set_preference('useAutomationExtension', False)
profile.update_preferences()
desired = DesiredCapabilities.FIREFOX
driver = webdriver.Firefox(options=options,
firefox_profile=profile,
desired_capabilities=desired)
log.info("New session is: %s %s" %
(driver.session_id, driver.command_executor._url))
return driver
def __init__(self,
firefox_profile=None,
firefox_binary=None,
timeout=30,
capabilities=None,
proxy=None):
self.binary = firefox_binary
self.profile = firefox_profile
if self.profile is None:
self.profile = FirefoxProfile()
self.profile.native_events_enabled = self.NATIVE_EVENTS_ALLOWED and self.profile.native_events_enabled
if self.binary is None:
self.binary = FirefoxBinary()
if capabilities is None:
capabilities = DesiredCapabilities.FIREFOX
if proxy is not None:
proxy.add_to_capabilities(capabilities)
RemoteWebDriver.__init__(self,
command_executor=ExtensionConnection(
"127.0.0.1", self.profile, self.binary,
timeout),
desired_capabilities=capabilities)
self._is_remote = False
