def start(cls, url, driver=""):
if cls.mode == "FireFox":
default_profile = {
'browser.usedOnWindows10': False,
'browser.usedOnWindows10.introURL': 'https://www.google.com/',
'startup.homepage_welcome_url.additional': 'about:blank',
'browser.startup.homepage_override.mstone': 'ignore',
}
profile = FirefoxProfile()
for name, value in default_profile.items():
profile.set_preference(name, value)
cls.driver = webdriver.Firefox(profile)
elif cls.mode == "Chrome":
try:
if driver != "":
chromedriver = driver
L.info(chromedriver)
os.environ["webdriver.chrome.driver"] = chromedriver
cls.driver = webdriver.Chrome(chromedriver)
else:
cls.driver = webdriver.Chrome()
except Exception as e:
L.warning(str(e))
raise SeleniumError(
"Can't find Selenium Driver Mode. %s" % cls.mode)
else:
raise SeleniumError(
"Can't find Selenium Driver Mode. %s" % cls.mode)
cls.driver.implicitly_wait(DEFAULT_WAIT)
cls.driver.set_window_size(WINDOW_SIZE_WIDTH, WINDOW_SIZE_HEIGHT)
cls.driver.get(url)
time.sleep(5)
def create(self) -> WebDriver:
pixel_density = current_app.config["WEBDRIVER_WINDOW"].get(
"pixel_density", 1)
if self._driver_type == "firefox":
driver_class = firefox.webdriver.WebDriver
options = firefox.options.Options()
profile = FirefoxProfile()
profile.set_preference("layout.css.devPixelsPerPx",
str(pixel_density))
kwargs: Dict[Any, Any] = dict(options=options,
firefox_profile=profile)
elif self._driver_type == "chrome":
driver_class = chrome.webdriver.WebDriver
options = chrome.options.Options()
options.add_argument(
f"--force-device-scale-factor={pixel_density}")
options.add_argument(
f"--window-size={self._window[0]},{self._window[1]}")
kwargs = dict(options=options)
else:
raise Exception(
f"Webdriver name ({self._driver_type}) not supported")
# Prepare args for the webdriver init
# Add additional configured options
for arg in current_app.config["WEBDRIVER_OPTION_ARGS"]:
options.add_argument(arg)
kwargs.update(current_app.config["WEBDRIVER_CONFIGURATION"])
logger.info("Init selenium driver")
return driver_class(**kwargs)
def openFF(self):
profile = FirefoxProfile()
profile.set_preference('network.http.phishy-userpass-length', 255)
profile.set_preference('network.automatic-ntlm-auth.trusted-uris',
'.sondait.com.br')
#profile.set_preference('dom.disable_beforeunload', 'false')
options = Options()
# options.set_headless(True)
# options.headless = True
# options.add_argument('-headless')
# options.add_argument("-headless") # Runs Chrome in headless mode.
# options.add_argument('-no-sandbox') # Bypass OS security model
# options.add_argument('-disable-gpu') # applicable to windows os only
# options.add_argument('start-maximized') #
# options.add_argument('disable-infobars')
# options.add_argument("-disable-extensions")
print("Headless FF initializing...")
self.driver = webdriver.Firefox(
executable_path='C:\Arris\geckodriver.exe',
options=options,
firefox_profile=profile)
if self.driver:
print("Headless FF initialized")
self.driver.implicitly_wait(10)
self.driver.delete_all_cookies()
return True
else:
print("Error FF not initialized")
return False
def prepare_browsers(headless: bool, driver_path: str,
twitter_profile_path: str) -> Browsers:
"""
Sets up browsers to search accounts
:param headless bool: Should search be performed in headless mode
:param driver_path: Path to geckodriver
:param twitter_profile_path: Path to twitter profile folder
:return: tuple of browsers, that are logged in LinkedIn and Xing
"""
logging.info("Running Twitter scraper from profile in %s",
twitter_profile_path)
driver_path = driver_path if driver_path else "geckodriver"
profile = FirefoxProfile()
twitter_profile = FirefoxProfile(twitter_profile_path)
twitter_profile.DEFAULT_PREFERENCES["frozen"][
"extensions.autoDisableScopes"] = 0
twitter_profile.set_preference("extensions.enabledScopes", 15)
logins = social_media_logins(driver_path, profile)
driver_options = FirefoxOptions()
driver_options.headless = headless
linked_in_driver = Firefox(options=driver_options,
firefox_profile=profile,
executable_path=driver_path)
xing_driver = Firefox(options=driver_options,
firefox_profile=profile,
executable_path=driver_path)
twitter_driver = Firefox(options=driver_options,
firefox_profile=twitter_profile,
executable_path=driver_path)
set_login_data(linked_in_driver, logins[0])
set_login_data(xing_driver, logins[1])
retoggleAllTheAddons(twitter_driver)
return Browsers(linked_in_driver, xing_driver, twitter_driver)
def create_driver():
#driver = webdriver.Chrome("chromedriver.exe")
custom_profile = FirefoxProfile()
custom_profile.set_preference("dom.disable_beforeunload", True)
driver = webdriver.Firefox(firefox_profile=custom_profile, executable_path="geckodriver.exe")
driver.implicitly_wait(10)
return driver
def get_user_page(url: str, profile_dir: str, outdir='.') -> None:
opt = FirefoxOptions()
# opt.add_argument('-headless')
p = FirefoxProfile(profile_dir)
p.set_preference('extensions.lastAppBuildId', '-1')
p.set_preference('network.proxy.socks', '')
p.set_preference('network.proxy.socks', '')
with Firefox(p, options=opt) as ff:
ff.get(url)
heights = []
while True:
count = 0
for elem in ff.find_elements_by_css_selector('.v1Nh3'):
try:
get_picture(ff, elem, outdir)
except Exception as e:
print(e)
heights.append(
ff.execute_script('return document.body.scrollHeight;'))
if len(heights) >= 3 and functools.reduce(
lambda a, b: a == b and a, heights[-3:]):
break
ff.execute_script(
'window.scrollTo(0, document.body.scrollHeight);')
time.sleep(2)
def _profile(self, arguments, extensions, proxy, user_agent):
""" Compile the capabilities of ChromeDriver inside the Container.
Args:
arguments (list):
extensions (list):
proxy (Proxy): unused.
user_agent (str):
Returns:
FirefoxProfile
"""
self.logger.debug('building browser profile')
profile = FirefoxProfile()
args = list(self.DEFAULT_ARGUMENTS)
if self.f(Flags.X_IMG):
args.append(('permissions.default.image', '2'))
if self.f(Flags.X_FLASH):
args.append(('dom.ipc.plugins.enabled.libflashplayer.so', 'false'))
for ext in extensions:
profile.add_extension(ext)
args.extend(arguments)
for arg_k, value in args:
profile.set_preference(arg_k, value)
if user_agent:
profile.set_preference('general.useragent.override', user_agent)
return profile
def main(cli=True, observations=2):
"""
Main code, including selenium driver initialization.
By default, outputs lines to stdout. If cli=False, the function returns a list of tuples with the resulting lines.
"""
try:
# XXX: remove this line to keep the log and continue from where you left off.
os.remove(LOG_PATH)
except OSError:
pass
profile = FirefoxProfile()
# The DesiredCapabilities approach does not work with firefox, see https://github.com/mozilla/geckodriver/issues/284
profile.set_preference("devtools.console.stdout.content", True)
options = Options()
# options.headless = True
driver = webdriver.Firefox(firefox_profile=profile, options=options)
try:
lines = collect(driver, observations)
if not cli:
return lines
for line in lines:
print(line)
return None
finally:
driver.close()
def __init__(self, browser_choice, card_to_search):
# Set up config file
try:
with open('config.yml', 'r') as config_file:
self.config = yaml.safe_load(config_file)
# Sanity check on card name
self.url = self.config[card_to_search]['url']
self.search_term = self.config[card_to_search]['search_term']
except KeyError:
print('{card} does not exist in the config file'.format(
card=card_to_search))
exit(1)
except IOError:
print('Config file is missing. Please download it from the source')
exit(1)
# Set up Selenium browser
if browser_choice == 'chrome':
chrome_options = ChromeOptions()
chrome_options.add_argument('--incognito')
self.browser = Chrome(chrome_options=chrome_options)
elif browser_choice == 'firefox':
firefox_profile = FirefoxProfile()
firefox_profile.set_preference('browser.privatebrowsing.autostart',
True)
self.browser = Firefox(firefox_profile=firefox_profile)
def generate_firefox_profile(
self: "SeleniumTestability",
preferences: OptionalDictType = None,
accept_untrusted_certs: bool = False,
proxy: OptionalStrType = None,
) -> FirefoxProfile:
"""
Generates a firefox profile that sets up few required preferences for SeleniumTestability to support all necessary features.
Parameters:
- ``preferences`` - firefox profile preferences in dictionary format.
- ``accept_untrusted_certs`` should we accept untrusted/self-signed certificates.
- ``proxy`` proxy options
Note: If you opt out using this keyword, you are not able to get logs with ``Get Logs`` and Firefox.
"""
profile = FirefoxProfile()
if preferences:
for key, value in preferences.items(): # type: ignore
profile.set_preference(key, value)
profile.set_preference("devtools.console.stdout.content", True)
profile.accept_untrusted_certs = accept_untrusted_certs
if proxy:
profile.set_proxy(proxy)
profile.update_preferences()
return profile
def __init__(self, url, usuario, senha):
# Configurar o navegador
profile = FirefoxProfile()
profile.set_preference("browser.download.panel.shown", False)
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", DIRETORIO)
profile.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"text/plain, application/octet-stream, application/binary,\
text/csv, application/csv, application/excel, text/comma-separated-values,\
text/xml, application/xml",
)
profile.accept_untrusted_certs = True
# # Instanciar o navegar firefox e inserir as configurações
browser = Remote(
# command_executor= 'http://127.0.0.1:4444/wd/hub',
desired_capabilities=DesiredCapabilities.FIREFOX,
browser_profile=profile,
)
# browser = Firefox(profile) # Utilizar para teste locais
# Maximizar a tela do navegador
browser.maximize_window()
# Inserir os atributos para classe pai com uma instancia do navegador
super().__init__(url, usuario, senha, browser)
def click_to_reveal_it(url, css_selector) -> webdriver.Firefox.page_source:
"""
Uses selenium to render request in a browser
locate a link, move mouse pointer to the link
and clicks it.
"""
BROWSER_ZOOM_SETTING = "layout.css.devPixelsPerPx"
ZOOM_FACTOR = "0.4"
TIME_SECS = 5
options = Options()
options.add_argument("-headless")
profile = FirefoxProfile()
profile.set_preference(BROWSER_ZOOM_SETTING, ZOOM_FACTOR)
browser = webdriver.Firefox(options=options, firefox_profile=profile)
browser.maximize_window()
browser.implicitly_wait(TIME_SECS)
browser.get(url)
try:
# TODO: add wait implementation (research: staleness_of)
time.sleep(1)
link = browser.find_element_by_css_selector(css_selector)
webdriver.ActionChains(browser).move_to_element(link).click(link).perform()
time.sleep(1)
return browser.page_source
except NoSuchElementException:
return browser.page_source
finally:
browser.quit()
def load_localstorage(self, session_id):
sessions = json.load(open(self.sessions_file))
storage_path = sessions[str(session_id)]["session_path"]
url = sessions[str(session_id)]["web_url"]
# Setting useragent to the same one the session saved with
useragent = sessions[str(session_id)]["useragent"]
profile = FirefoxProfile()
profile.set_preference("general.useragent.override", useragent)
localStorage = pickle.load(open(storage_path, "rb"))
try:
browser = Firefox(profile)
except:
error("Couldn't open browser to view session!")
return
browser.get(url)
browser.delete_all_cookies()
browser.execute_script(
"window.localStorage.clear()") # clear the current localStorage
for key, value in localStorage.items():
browser.execute_script(
"window.localStorage.setItem(arguments[0], arguments[1]);",
key, value)
status(f"Session {session_id} loaded")
browser.refresh()
self.browsers.append(browser)
def launch_browser(headless=False, timeout=4):
"""Launch a Firefox webdriver with disabled notifications,
allow page loading, optionally phantom mode
Args:
headless (bool): if True, launch in phantom/headless mode,
where you cannot see the browser (default=False)
timeout (int): time (s) to wait for a response (default=10)
Returns:
Webdriver (object): Firefox. With firefox profile settings
as explained above.
"""
# https://stackoverflow.com/questions/32953498/how-can-i-remove-notifications-and-alerts-from-browser-selenium-python-2-7-7
# https://stackoverflow.com/questions/26566799/how-to-wait-until-the-page-is-loaded-with-selenium-for-python
# https://stackoverflow.com/questions/5370762/how-to-hide-firefox-window-selenium-webdriver
# https://developer.mozilla.org/en-US/Firefox/Headless_mode#Selenium-in-Python
options = Options()
if headless:
options.add_argument('-headless')
logger.debug('Running in headless mode!')
fp = FirefoxProfile()
fp.set_preference("dom.webnotifications.enabled", False)
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.dir", os.getcwd())
fp.set_preference("browser.helperApps.neverAsk.saveToDisk",
"application/zip")
web_browser = Firefox(firefox_profile=fp,
executable_path='geckodriver',
options=options)
web_browser.implicitly_wait(timeout)
return web_browser
def run_local(self, browser_version):
"""
Return the local driver
: TODO Add support to other browsers
"""
if command_line.get_browser().lower() == "chrome":
options = chromeOptions()
options.add_argument("disable-infobars")
if command_line.headless() == "y":
options.add_argument("--headless") # For Headless chrome
caps = DesiredCapabilities.CHROME
local_driver = webdriver.Chrome(options=options,
desired_capabilities=caps)
return local_driver
if command_line.get_browser().lower() == "ff" \
or command_line.get_browser().lower() == "firefox":
profile = FirefoxProfile()
options = firefoxOptions()
profile.set_preference("geo.prompt.testing", True)
profile.set_preference("geo.prompt.testing.allow", True)
if command_line.headless() == "y":
options.headless = True # For Headless firefox
local_driver = webdriver.Firefox(
executable_path=GeckoDriverManager().install(),
firefox_profile=profile,
options=options)
return local_driver
def _create_profile(self):
"""Create profile object.
"""
profile = FirefoxProfile()
for name, value in self.default_profile.items():
profile.set_preference(name, value)
return profile
def _buildFirefoxDriver(self, headless=False):
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
profile.headless = headless
profile.set_preference('security.fileuri.strict_origin_policy', False)
o = Options()
o.set_headless(self.config_options['headless'])
return KrFirefox(firefox_profile=profile, options=o)
def _connection():
o = Options()
fp = FirefoxProfile()
o.add_argument('-private')
fp.set_preference("network.proxy.type", 0)
#fp.set_preference("general.useragent.override","iPhone")
fp.update_preferences()
return Firefox(firefox_profile=fp, firefox_options=o)
def __init__(self):
profile = FirefoxProfile()
profile.set_preference("dom.webdriver.enabled", False)
profile.set_preference('useAutomationExtension', False)
profile.update_preferences()
desired = DesiredCapabilities.FIREFOX
super().__init__(firefox_profile=profile, desired_capabilities=desired)
def __init__(self, delay=1, browser="firefox"):
"""delay: Number of extra seconds to wait when a page is
supposedly loaded. Try raising this in case of weird errors.
browser: `firefox` or `chrome`. The ChromeDriver executable for your
OS must be inside the bin directory for Chrome to work. Get it from:
http://chromedriver.storage.googleapis.com/index.html
"""
self.extra_delay = delay # extra time to wait after each operation (s)
self.temp_dir = mkdtemp()
self.vdisplay = Xvfb()
self.vdisplay.start()
def get_bin_path(subdir):
path = os_path.dirname(os_path.realpath(__file__)) + os_sep
return path + os_sep.join([pardir, "bin", subdir])
if browser == "firefox":
profile = FirefoxProfile()
# Open links in same window
profile.set_preference("browser.link.open_newwindow", 1)
# Download to temp dir, for files we can't open inline
profile.set_preference("browser.download.dir", self.temp_dir)
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting",
"False")
profile.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"application/msword, application/vnd.ms-word, application/rtf, application/octet-stream"
)
# Add extension for overriding Content-Disposition headers, etc
extensions_dir = get_bin_path('firefox-plugins-enabled')
for filename in listdir(extensions_dir):
fullfilename = os_sep.join([extensions_dir, filename])
profile.add_extension(extension=fullfilename)
driver = Firefox(profile)
elif browser == "chrome":
# Add extension for overriding Content-Disposition headers
options = ChromeOptions()
options.add_extension(get_bin_path('bin/undisposition.crx'))
driver = Chrome(executable_path=get_bin_path('bin/chromedriver'),
chrome_options=options)
else:
raise Exception("Not a valid browser name")
self.selenium_driver = EventFiringWebDriver(driver, CustomListener())
"""selenium_driver is a EventFiringWebDriver, so that it can
trigger javascript event
"""
self.browser_version = " ".join([
self.selenium_driver.capabilities['browserName'],
self.selenium_driver.capabilities['version']
]) # 'Firefox 33.0'
def get_search_results(minresults=40):
"""Collect property urls and types by going through the search result pages of new houses and appartments,
stopping when having reached the minimum number of results and returning a dictionary of {'url1':True/False, 'url2':True/False, ...}.
True means house. False means apartment. Without argument only the first page is collected (~60 results)"""
search_results = {}
result_count = 0
# set on which page to start the search
page_number = 1
options = FirefoxOptions()
options.add_argument('-headless')
options.set_preference("dom.webdriver.enabled", False)
profile = FirefoxProfile('src/scraping/bolzyxyb.heroku')
profile.set_preference('useAutomationExtension', False)
driver = Firefox(firefox_binary='usr/lib/firefox/firefox',
options=options,
firefox_profile=profile)
driver.implicitly_wait(15)
# start the progress indicator and timeout logic
start_time = time.monotonic()
time_spent = 0
while result_count < minresults and time_spent < 1800:
# for each loop, scrape one results page of houses and one of appartments
# the results are added if they are not there yet
for houselink in get_page_urls(pagenr=page_number,
kind="house",
drv=driver):
if houselink not in search_results:
search_results[houselink] = True
for apartmentlink in get_page_urls(pagenr=page_number,
kind="apartment",
drv=driver):
if apartmentlink not in search_results:
search_results[apartmentlink] = False
result_count = len(search_results)
page_number += 1
# update progress indicator
time_spent = time.monotonic() - start_time
total_time_estimation = 1 / (result_count / minresults) * time_spent
if total_time_estimation > 1800:
capped_time = 1800
else:
capped_time = total_time_estimation
time_remaining = capped_time - time_spent
print(f"Finishing in {time_remaining/60:.1f} minutes")
driver.close()
print("Finished")
return search_results
def __init__(self, cer_data, key_data):
self.temp_cer = self.temp_file(cer_data)
self.temp_key = self.temp_file(key_data)
profile = FirefoxProfile(settings.PROFILE_PATH)
for preference in settings.PROFILE_PREFERENCES:
profile.set_preference(*preference)
self._driver = Firefox(firefox_profile = profile)
self._driver.get(settings.GET_CFDI_URL)
def get_cookies(num):
# user = '******'
# pwd = 'lxq69688'
n = num % 2
users = ['[email protected]', '14785107068----lxq69688']
user = users[n]
one_user = (user).split('----')
user, pwd = one_user[0], one_user[1]
option = FirefoxProfile()
option.set_preference("dom.webnotifications.enabled", False)
browser = Firefox(option)
wait = WebDriverWait(browser, 120)
browser.delete_all_cookies()
browser.get(start_url)
submit = wait.until(
EC.presence_of_element_located(
(By.XPATH,
'//div[@class="W_login_form"]/div[@class="info_list login_btn"]/a'
)))
sleep(1)
submit.click()
user_element = wait.until(
EC.presence_of_element_located((By.XPATH, '//input[@id="loginname"]')))
# user_element.clear()
sleep(1)
user_element.send_keys(user)
print('账号:', user)
pwd_element = wait.until(
EC.presence_of_element_located(
(By.XPATH, '//input[@type="password"]')))
# pwd_element.clear()
sleep(1)
pwd_element.send_keys(pwd)
print('密码:', pwd)
submit = wait.until(
EC.presence_of_element_located(
(By.XPATH,
'//div[@class="W_login_form"]/div[@class="info_list login_btn"]/a'
)))
submit.click()
print('点击登陆！')
# sleep(2)
user_btn = wait.until(
EC.presence_of_element_located(
(By.XPATH, '//ul[@class="gn_nav_list"]/li[5]//em[2]')))
print('登陆用户：', user_btn.text)
#条件 确认是否登陆
cookies = browser.get_cookies()
cookie = ';'.join(i['name'] + '=' + i['value'] for i in cookies)
browser.close()
cookies = {
'Cookie': cookie,
}
return cookies
def __init__(self):
profile = FirefoxProfile()
profile.set_preference('media.volume_scale', '0.0') # Mute audio
self.driver = Firefox(firefox_profile=profile)
self.clients = None
self.players = None
self.join_link = None
self.state = GameState.STOPPED
self.quit_flag = False
def __call__(self, config, args):
profile = FirefoxProfile()
if config.user_agent:
profile.set_preference("general.useragent.override",
config.user_agent)
if config.ignore_ssl:
profile.accept_untrusted_certs = True
args["firefox_profile"] = profile
args["capabilities"] = args["desired_capabilities"]
args.pop("desired_capabilities")
return args
def __call__(self, config, args):
profile = FirefoxProfile()
if config.user_agent:
profile.set_preference("general.useragent.override",
config.user_agent)
if config.ignore_ssl:
profile.accept_untrusted_certs = True
args["firefox_profile"] = profile
args["capabilities"] = args["desired_capabilities"]
args.pop("desired_capabilities")
return args
def clear_proxy(profile=None):
"""
:param profile:
:return:
"""
if profile is None:
profile = FirefoxProfile()
profile.set_preference("network.proxy.type", 0)
profile.update_preferences()
return profile
def test_that_we_can_accept_a_profile(capabilities, webserver):
profile1 = FirefoxProfile()
profile1.set_preference("browser.startup.homepage_override.mstone", "")
profile1.set_preference("startup.homepage_welcome_url",
webserver.where_is('simpleTest.html'))
profile1.update_preferences()
profile2 = FirefoxProfile(profile1.path)
driver = Firefox(capabilities=capabilities, firefox_profile=profile2)
title = driver.title
driver.quit()
assert "Hello WebDriver" == title
def start_firefox(proxy=None, user_agent=None):
p = FirefoxProfile("E://profiles")
p.accept_untrusted_certs = True
if user_agent:
p.set_preference("general.useragent.override", user_agent)
capabilities = DesiredCapabilities().FIREFOX.copy()
capabilities['acceptInsecureCerts'] = True
if proxy:
p.set_proxy(proxy)
browser = Firefox(firefox_profile=p, capabilities=capabilities)
return browser
def browser_setup():
"""Set a driver object."""
cfg = get_config()["browser"]
profile = FirefoxProfile()
user_agent = UserAgent()
profile.native_events_enabled = cfg["native_events_enabled"]
profile.set_preference("general.useragent.override", user_agent.random)
opts = FirefoxOptions()
opts.headless = cfg["headless"]
driver = Firefox(options=opts, executable_path=cfg["gecko_driver"])
driver.implicitly_wait(cfg["timeout"])
return driver
def set_browser(self, browser):
# CHROME
if browser == 'Chrome':
options = ChromeOptions();
options.add_argument("--disable-notifications");
self.browser = Chrome(options=options)
# FIREFOX
if browser == 'Firefox':
profile = FirefoxProfile();
profile.set_preference("dom.webnotifications.enabled", False);
self.browser = Firefox(firefox_profile=profile)
def get_firefox():
caps = DesiredCapabilities().FIREFOX
options = FirefoxOptions()
options.add_argument("--headless")
caps["pageLoadStrategy"] = "eager" # interactive
profile = FirefoxProfile()
profile.set_preference("dom.disable_beforeunload", True)
browser = Firefox(desired_capabilities=caps,
firefox_profile=profile,
options=options)
browser.set_page_load_timeout(6)
return browser
def create_driver(browser="chrome"):
if browser == "chrome":
driver = webdriver.Chrome("./driver/chromedriver.exe")
else:
custom_profile = FirefoxProfile()
custom_profile.set_preference("dom.disable_beforeunload", True)
driver = webdriver.Firefox(firefox_profile=custom_profile, executable_path="./driver/geckodriver.exe")
driver.implicitly_wait(10)
return driver
def test_that_we_can_accept_a_profile(capabilities, webserver):
profile1 = FirefoxProfile()
profile1.set_preference("browser.startup.homepage_override.mstone", "")
profile1.set_preference("startup.homepage_welcome_url", webserver.where_is('simpleTest.html'))
profile1.update_preferences()
profile2 = FirefoxProfile(profile1.path)
driver = Firefox(
capabilities=capabilities,
firefox_profile=profile2)
title = driver.title
driver.quit()
assert "Hello WebDriver" == title
def test_proxy(proxy):
if not proxy:
return
# r = get('https://www.google.com/', proxy)
ip, port, version = proxy.split(',')
print(f'Checking proxy {ip}:{port}')
options = Options()
options.headless = True
profile = FirefoxProfile()
profile.set_preference('network.proxy.type', 1)
profile.set_preference("network.proxy.socks", ip)
profile.set_preference("network.proxy.socks_port", int(port))
profile.set_preference("network.proxy.socks_version", int(version))
profile.update_preferences()
browser = Firefox(executable_path=r"./geckodriver",
firefox_profile=profile,
options=options)
try:
browser.get('https://showip.net/')
soup = BeautifulSoup(browser.page_source, 'html.parser')
return soup.select_one('#checkip').get('value').strip() == proxy.split(
',')[0].strip()
except KeyboardInterrupt:
raise KeyboardInterrupt('Abort')
except:
logging.error(traceback.format_exc())
finally:
browser.quit()
def generate_profile(useragent="(default)"):
profile = FirefoxProfile()
if useragent.strip().lower()=="(default)":
status("Using the default useragent")
return profile
elif useragent.strip().lower()=="(random)":
random_useragent = generate_user_agent(os=('mac', 'linux'))
profile.set_preference("general.useragent.override", random_useragent) # To make our useragent random
status("Using random useragent "+random_useragent)
return profile
else:
profile.set_preference("general.useragent.override", useragent)
status("Using useragent "+useragent)
return profile
def setUp(self):
if _platform != "darwin":
self.display = Display(visible=0, size=(1280, 800))
self.display.start()
p = FirefoxProfile()
p.set_preference("webdriver.log.file", "/tmp/firefox_console")
self.driver = webdriver.Firefox(p)
self.base_url = os.getenv('SELENIUM_BASE_URL')
self.verificationErrors = []
self.accept_next_alert = True
self.driver.set_window_size(1280, 800)
self.project_name = "DBaaS"
def firefox_profile(request):
profile = FirefoxProfile(request.config.getoption('firefox_profile'))
for preference in request.config.getoption('firefox_preferences'):
name, value = preference
if value.isdigit():
# handle integer preferences
value = int(value)
elif value.lower() in ['true', 'false']:
# handle boolean preferences
value = value.lower() == 'true'
profile.set_preference(name, value)
profile.update_preferences()
for extension in request.config.getoption('firefox_extensions'):
profile.add_extension(extension)
return profile
def __init__(self, delay=1, browser="firefox"):
"""delay: Number of extra seconds to wait when a page is
supposedly loaded. Try raising this in case of weird errors.
browser: `firefox` or `chrome`. The ChromeDriver executable for your
OS must be inside the bin directory for Chrome to work. Get it from:
http://chromedriver.storage.googleapis.com/index.html
"""
self.extra_delay = delay # extra time to wait after each operation (s)
self.temp_dir = mkdtemp()
self.vdisplay = Xvfb()
self.vdisplay.start()
if browser == "firefox":
profile = FirefoxProfile()
# Open links in same window
profile.set_preference("browser.link.open_newwindow", 1)
# Download to temp dir, for files we can't open inline
profile.set_preference("browser.download.dir", self.temp_dir)
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting",
"False")
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
"application/msword, application/vnd.ms-word, application/rtf, application/octet-stream")
# Add extension for overriding Content-Disposition headers, etc
extensions_dir = os_sep.join(['bin', 'firefox-plugins-enabled'])
for filename in listdir(extensions_dir):
fullfilename = os_sep.join([extensions_dir, filename])
profile.add_extension(extension=fullfilename)
driver = Firefox(profile)
elif browser == "chrome":
# Add extension for overriding Content-Disposition headers
options = ChromeOptions()
options.add_extension('bin/undisposition.crx')
driver = Chrome(executable_path='bin/chromedriver',
chrome_options=options)
else:
raise Exception("Not a valid browser name")
self.selenium_driver = EventFiringWebDriver(driver, CustomListener())
"""selenium_driver is a EventFiringWebDriver, so that it can
trigger javascript event
"""
self.browser_version = " ".join([
self.selenium_driver.capabilities['browserName'],
self.selenium_driver.capabilities['version']]) # 'Firefox 33.0'
def setup_firefox_profile(self):
"""Return a custom firefox profile with given preferences
and extensions.
"""
fp = FirefoxProfile()
if self.extensions:
# Load extensions.
for ext in self.extensions:
fp.add_extension(ext)
if self.preferences:
# Load preferences
for key, value in self.preferences:
fp.set_preference(key, value)
return fp
def test_that_unicode_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference('sample.preference.2', unicode('hi there'))
profile.update_preferences()
assert 'hi there' == profile.default_preferences["sample.preference.2"]
encoded = profile.encoded
decoded = base64.b64decode(encoded)
with BytesIO(decoded) as fp:
zip = zipfile.ZipFile(fp, "r")
for entry in zip.namelist():
if entry.endswith('user.js'):
user_js = zip.read(entry)
for line in user_js.splitlines():
if line.startswith(b'user_pref("sample.preference.2",'):
assert line.endswith(b'hi there");')
# there should be only one user.js
break
def load_cookie(self, session_id):
sessions = json.load(open( self.sessions_file ))
cookie_path = sessions[str(session_id)]["session_path"]
url = sessions[str(session_id)]["web_url"]
# Setting useragent to the same one the session saved with
useragent = sessions[str(session_id)]["useragent"]
profile = FirefoxProfile()
profile.set_preference("general.useragent.override", useragent )
cookies = pickle.load(open(cookie_path, "rb"))
try:
browser = Firefox(profile)
except:
error("Couldn't open browser to view session!")
return
browser.get(url)
browser.delete_all_cookies()
browser.execute_script("window.localStorage.clear()") # clear the current localStorage
for cookie in cookies:
browser.add_cookie(cookie)
status(f"Session {session_id} loaded")
browser.refresh()
self.browsers.append(browser)
def firefox_profile(pytestconfig):
profile = None
if pytestconfig.getoption('firefox_profile'):
profile = FirefoxProfile(pytestconfig.getoption('firefox_profile'))
warnings.warn(
'--firefox-profile has been deprecated and will be removed in '
'a future release. Please use the firefox_options fixture to '
'set a profile path or FirefoxProfile object using '
'firefox_options.profile.', DeprecationWarning)
if pytestconfig.getoption('firefox_preferences'):
profile = profile or FirefoxProfile()
warnings.warn(
'--firefox-preference has been deprecated and will be removed in '
'a future release. Please use the firefox_options fixture to set '
'preferences using firefox_options.set_preference. If you are '
'using Firefox 47 or earlier then you will need to create a '
'FirefoxProfile object with preferences and set this using '
'firefox_options.profile.', DeprecationWarning)
for preference in pytestconfig.getoption('firefox_preferences'):
name, value = preference
if value.isdigit():
# handle integer preferences
value = int(value)
elif value.lower() in ['true', 'false']:
# handle boolean preferences
value = value.lower() == 'true'
profile.set_preference(name, value)
profile.update_preferences()
if pytestconfig.getoption('firefox_extensions'):
profile = profile or FirefoxProfile()
warnings.warn(
'--firefox-extensions has been deprecated and will be removed in '
'a future release. Please use the firefox_options fixture to '
'create a FirefoxProfile object with extensions and set this '
'using firefox_options.profile.', DeprecationWarning)
for extension in pytestconfig.getoption('firefox_extensions'):
profile.add_extension(extension)
return profile
def __init__(self, should_hide_window, installation_path):
# ASSUMPTION: installation_path refers to a valid firefox executable.
firefox_binary = None if installation_path is None else FirefoxBinary(installation_path)
profile = FirefoxProfile()
profile.set_preference("intl.accept_languages", "en-us,en")
# do not load images:
profile.set_preference('browser.migration.version', 9001)
profile.set_preference('permissions.default.image', 2)
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
driver = selenium.webdriver.Firefox(firefox_profile=profile, firefox_binary=firefox_binary)
web_driver = EnhancedWebDriver(driver, should_hide_window)
super(FacebookScrapingWebDriver, self).__init__(web_driver, should_hide_window)
self.implicitly_wait(5)
def browser(request):
"""
initialize the selenium test case
"""
profile = FirefoxProfile()
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", DOWNLOAD_DIR)
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv")
b = Firefox(firefox_profile=profile)
b.implicitly_wait(10)
b.maximize_window()
request.addfinalizer(lambda *args: b.quit())
yield b
time.sleep(10)
def test_that_integer_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference("sample.int.preference", 12345)
profile.update_preferences()
assert 12345 == profile.default_preferences["sample.int.preference"]
default_profile = {
'security.warn_entering_secure': False,
'security.warn_entering_secure.show_once': True,
'security.warn_entering_weak': False,
'security.warn_entering_weak._show_once': True,
'security.warn_leaving_secure': False,
'security.warn_leaving_secure.show_once': True,
'security.warn_leaving_weak': False,
'security.warn_leaving_weak._show_once': True,
'security.warn_submit_insecure': False,
'security.warn_viewing_mixed': False,
'security.warn_viewing_mixed.show_once': True,
}
profile = FirefoxProfile()
for name, value in default_profile.items():
profile.set_preference(name, value)
proxy = Proxy()
proxy.ftp_proxy = proxy.ssl_proxy = proxy.http_proxy = None
browser = Firefox(firefox_profile=profile, proxy=proxy)
tokyo_url = 'https://ticketcamp.net/venue/tokyo/' #東京に接続
browser.get(tokyo_url)
'''
'''
url_list = queue.Queue()
def makeProfile(self, config):
profile = FirefoxProfile()
# Disable Firefox auto update
profile.set_preference("app.update.enabled", False)
try:
if config["fire_bug"]:
profile.add_extension(FIREBUG_EXTENSION)
domain = "extensions.firebug."
# Set default Firebug preferences
# Avoid Firebug start page
profile.set_preference(domain + "currentVersion", "2.0.6")
# Activate everything
profile.set_preference(domain + "allPagesActivation", "on")
profile.set_preference(domain + "defaultPanelName", "net")
# Enable Firebug on all sites
profile.set_preference(domain + "net.enableSites", True)
self.logger.info("Firebug profile settings enabled")
except KeyError:
self.logger.warning("Firebug profile settings failed")
pass
try:
if config["net_export"]:
profile.add_extension(NETEXPORT_EXTENSION)
# Set default NetExport preferences
self.har_output = config["net_export_output"]
#self.logger.info("Output HAR directory: {0}".format(self.har_output))
profile.set_preference(domain + "netexport.defaultLogDir", self.har_output)
profile.set_preference(domain + "netexport.autoExportToFile", True)
profile.set_preference(domain + "netexport.alwaysEnableAutoExport", True)
# Do not show preview output
profile.set_preference(domain + "netexport.showPreview", True)
profile.set_preference(domain + "netexport.pageLoadedTimeout", 3000)
# Log dir
self.logger.info("NetExport profile settings enabled.")
# Har ID to check file exists
self.har_id = int((datetime.datetime.now()-datetime.datetime(1970,1,1)).total_seconds()*1000000)
self.url += "/?{0}".format(self.har_id)
#print self.url
except KeyError:
self.logger.warning("NetExport profile settings failed")
pass
profile.update_preferences()
return profile
class Action(object):
def __init__(self):
if config.USE_HEADLESS:
self.driver = webdriver.PhantomJS()
else:
self.profile = FirefoxProfile()
self.profile.set_preference("security.tls.version.fallback-limit", 0)
self.profile.set_preference("security.tls.version.min", 0)
self.driver = webdriver.Firefox(self.profile)
self.driver.set_window_size(1600, 1200)
self.driver.implicitly_wait(30)
self.verificationErrors = []
self.accept_next_alert = True
def _close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally: self.accept_next_alert = True
def _wait_for_element_id(self, id):
sofar = 0
while True:
try:
self.driver.find_element_by_id(id)
except:
time.sleep(0.5)
sofar += 0.5
if sofar > 20:
raise TimeoutException
else:
break
def _wait_for_not_visible(self, id):
sofar = 0
while True:
try:
elem = self.driver.find_element_by_id(id)
if not elem.is_displayed():
break
else:
time.sleep(0.5)
sofar += 0.5
if sofar > 20:
raise TimeoutException
except:
pass
def is_at_home(self):
"""
Checks whether we are currently in the app home page, and thus able to start adding entries.
:return: True if we are in the app home page, false otherwise.
:rtype: bool
"""
try:
self.driver.find_element_by_id("B16381221745222794") # Add new button
except:
return False
else:
return True
def wait_for_home(self):
"""
Waits until we are at the home screen and thus able to add new entries.
:return:
"""
self._wait_for_element_id("B16381221745222794")
def login(self):
"""
Logins to the app if not alaready logged in.
:return:
"""
print "Logging in... "
driver = self.driver
driver.get(START_URL)
driver.find_element_by_name("ssousername").clear()
driver.find_element_by_name("ssousername").send_keys(config.DEUSTO_USERNAME)
driver.find_element_by_name("password").clear()
driver.find_element_by_name("password").send_keys(config.DEUSTO_PASSWORD)
driver.find_element_by_css_selector("input[type=\"submit\"]").click()
self._wait_for_element_id("B16381221745222794")
print " done."
def add_entry(self, project, unit, concept, date, hours):
print "Adding entry... "
driver = self.driver
# Click the add new btn
driver.find_element_by_id("B16381221745222794").click()
Select(driver.find_element_by_id("P6_PA_COD_PROYECTO")).select_by_visible_text(project)
Select(driver.find_element_by_id("P6_PA_COD_AREA_ACTIVIDAD")).select_by_visible_text(unit)
Select(driver.find_element_by_id("P6_PA_COD_TIPO_TRABAJO")).select_by_visible_text(concept)
driver.find_element_by_css_selector("img.ui-datepicker-trigger").click()
driver.find_element_by_css_selector("span.ui-icon.ui-icon-circle-triangle-w").click()
driver.find_element_by_css_selector("span.ui-icon.ui-icon-circle-triangle-w").click()
driver.find_element_by_css_selector("span.ui-icon.ui-icon-circle-triangle-w").click()
driver.find_element_by_css_selector("span.ui-icon.ui-icon-circle-triangle-w").click()
driver.find_element_by_id("P6_PA_FECHA").clear()
driver.find_element_by_id("P6_PA_FECHA").send_keys(date)
driver.find_element_by_css_selector("div.uRegionContent.clearfix").click()
driver.find_element_by_id("P6_HORAS").clear()
driver.find_element_by_id("P6_HORAS").send_keys(hours)
driver.find_element_by_id("B16375215916222755").click()
self.wait_for_home()
print " done."
def add_entries(self, entries):
"""
Massively adds the specified entries.
Entries is a list of dicts.
Example:
[
{
"project": "GO-LAB",
"unit": "Unidad Internet",
"concept": "I+D Desarrollo Proyecto",
"date": "1/1/2014",
"hours": "1"
}
]
:param entries:
:return:
"""
for entry in entries:
self.add_entry(**entry) # TO-DO: Not too pretty, improve this, add some error-tolerance.
def add_entries_safe(self, entries, progress_file):
"""
Massively adds the specified entries, just like add_entries.
Successfully added entries are removed from the dictionary,
and saved to progress_file (path).
:param entries:
:param progress_file: Path to the file in which to save remaining entries.
:return:
"""
total = len(entries)
fails = 0
added = 0
# Remember each entry in a dict.
i = 0
d = OrderedDict()
for entry in entries:
d[i] = entry
i += 1
i = 0
for entry in entries:
try:
self.add_entry(**entry) # TO-DO: Not too pretty, improve this, add some error-tolerance.
print "[SUCCESS] Registered entry: %r" % (entry)
added += 1
except:
print "[FAIL] FAILED TO ADD ENTRY: %r" % (entry)
fails += 1
else:
del d[i]
# Save the current remaining dictionary to disk.
raw = json.dumps({"entries": d.values()})
f = file(progress_file, "w")
f.write(raw)
f.close()
i += 1
print "[PROGRESS]: %d out of %d. Errors: %d" % (added, total, fails)
def remove_all_existing(self):
"""
Removes every existing entry.
:return:
"""
driver = self.driver
# This is mainly to support this function in PhantomJS (which doesn't support confirms)
driver.execute_script("window.confirm = function(msg) { return true; }")
# Display every single entry in one page.
driver.execute_script("gReport.search('SEARCH', 1000000)")
self._wait_for_not_visible("apexir_LOADER")
# Mark them all for deletion.
driver.execute_script("""$("input[type=checkbox]").attr("checked", "checked")""")
self._wait_for_element_id("B24879923011698259")
driver.find_element_by_id("B24879923011698259").click()
# self._close_alert_and_get_its_text()
self.wait_for_home()
def profile():
profile = FirefoxProfile()
profile.set_preference('browser.startup.homepage_override.mstone', '')
profile.set_preference('startup.homepage_welcome_url', 'about:')
profile.update_preferences()
return profile
class DynModals(unittest.TestCase):
def setUp(self):
reset_database()
if os.environ.get("SELENIUM_HEADLESS"):
self.driver = webdriver.PhantomJS()
else:
self.profile = FirefoxProfile();
self.profile.set_preference("intl.accept_languages", "en")
self.driver = webdriver.Firefox(self.profile)
self.driver.set_window_size(1600, 1200)
self.driver.implicitly_wait(30)
self.base_url = "http://*****:*****@type='submit']").click()
driver.find_element_by_xpath(u"(//a[contains(text(),'Start composing »')])[2]").click()
for i in range(60):
try:
if self.is_element_present(By.CSS_SELECTOR, "td.sorting_1"): break
except: pass
time.sleep(1)
else: self.fail("time out")
time.sleep(1)
driver.find_element_by_css_selector("td.sorting_1").click()
time.sleep(1)
driver.find_element_by_id("sendurlbtn").click()
driver.find_element_by_css_selector("input.btn.btn-primary").click()
driver.find_element_by_css_selector("input.btn.btn-success").click()
driver.find_element_by_link_text("Apps").click()
self.assertEqual("Concept Mapper", driver.find_element_by_css_selector("h3.app-title.dyn-changeable").text)
driver.find_element_by_css_selector("h3.app-title.dyn-changeable").click()
time.sleep(0.5)
driver.find_element_by_id("appname-field").clear()
driver.find_element_by_id("appname-field").send_keys("The Best Concept Mapper")
driver.find_element_by_css_selector("button.btn.btn-primary").click()
time.sleep(1)
self.assertEqual("Apps - User Profile", driver.title)
self.assertEqual("The Best Concept Mapper", driver.find_element_by_css_selector("h3.app-title.dyn-changeable").text)
driver.find_element_by_id("desclabel").click()
time.sleep(0.5)
driver.find_element_by_id("appdesc-field").clear()
driver.find_element_by_id("appdesc-field").send_keys("The description for the app")
driver.find_element_by_css_selector("#appdesc-modal > div.modal-dialog > div.modal-content > form > div.modal-footer > button.btn.btn-primary").click()
time.sleep(0.5)
self.wait_until_equals("The description for the app", lambda: driver.find_element_by_id("descfield").text)
driver.find_element_by_link_text("Open").click()
time.sleep(0.5)
self.assertEqual("App Composer :: Edit the app content", driver.title)
self.assertEqual("Adapt - The Best Concept Mapper", driver.find_element_by_id("apptitle").text)
driver.find_element_by_id("apptitle").click()
time.sleep(0.5)
driver.find_element_by_id("appname-field").clear()
driver.find_element_by_id("appname-field").send_keys("The new name for Concept Mapper")
driver.find_element_by_css_selector("button.btn.btn-primary").click()
time.sleep(2)
self.assertEqual("App Composer :: Edit the app content", driver.title)
self.assertEqual("Adapt - The new name for Concept Mapper", driver.find_element_by_id("apptitle").text)
driver.find_element_by_link_text("Apps").click()
driver.find_element_by_link_text("Delete").click()
driver.find_element_by_name("delete").click()
driver.find_element_by_xpath(u"(//a[contains(text(),'Start composing »')])[3]").click()
for i in range(60):
try:
if "The Concept Mapper tool lets you cre..." == driver.find_element_by_css_selector("p.p1").text: break
except: pass
time.sleep(1)
else: self.fail("time out")
time.sleep(1)
driver.find_element_by_css_selector("p.p1").click()
driver.find_element_by_id("sendurlbtn").click()
driver.find_element_by_link_text("Apps").click()
driver.find_element_by_link_text("Open").click()
driver.find_element_by_id("appfullname").click()
time.sleep(1)
driver.find_element_by_id("appname-field").clear()
driver.find_element_by_id("appname-field").send_keys("My Concept Mapper")
driver.find_element_by_css_selector("button.btn.btn-primary").click()
time.sleep(1)
self.wait_until_equals("My Concept Mapper", lambda : driver.find_element_by_id("appfullname").text)
driver.find_element_by_link_text("Apps").click()
self.assertEqual("My Concept Mapper", driver.find_element_by_css_selector("h3.app-title.dyn-changeable").text)
driver.find_element_by_css_selector("h3.app-title.dyn-changeable").click()
time.sleep(0.5)
driver.find_element_by_id("appname-field").clear()
driver.find_element_by_id("appname-field").send_keys("My Concept Mapper Name")
driver.find_element_by_css_selector("button.btn.btn-primary").click()
time.sleep(0.2)
self.wait_until_equals("My Concept Mapper Name", lambda : driver.find_element_by_css_selector("h3.app-title.dyn-changeable").text)
driver.find_element_by_id("descfield").click()
time.sleep(0.5)
driver.find_element_by_id("appdesc-field").clear()
driver.find_element_by_id("appdesc-field").send_keys("This is quite a great Concept Mapper")
driver.find_element_by_css_selector("#appdesc-modal > div.modal-dialog > div.modal-content > form > div.modal-footer > button.btn.btn-primary").click()
time.sleep(0.2)
self.wait_until_equals("This is quite a great Concept Mapper", lambda : driver.find_element_by_id("descfield").text)
driver.find_element_by_link_text("Delete").click()
driver.find_element_by_name("delete").click()
driver.find_element_by_link_text("Log out").click()
def is_element_present(self, how, what):
try: self.driver.find_element(by=how, value=what)
except NoSuchElementException, e: return False
return True
def is_alert_present(self):
try: self.driver.switch_to_alert()
except NoAlertPresentException, e: return False
return True
def close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally: self.accept_next_alert = True
def tearDown(self):
self.driver.quit()
self.assertEqual([], self.verificationErrors)
class BasicAdapt(unittest.TestCase):
def setUp(self):
if os.environ.get("SELENIUM_HEADLESS"):
self.driver = webdriver.PhantomJS()
else:
self.profile = FirefoxProfile();
self.profile.set_preference("intl.accept_languages", "en")
self.driver = webdriver.Firefox(self.profile)
self.driver.set_window_size(1600, 1200)
self.driver.implicitly_wait(30)
self.base_url = "http://*****:*****@type='submit']").click()
driver.find_element_by_xpath(u"(//a[contains(text(),'Start composing »')])[2]").click()
for i in range(60):
try:
if self.is_element_present(By.CSS_SELECTOR, "td.sorting_1"): break
except: pass
time.sleep(1)
else: self.fail("time out")
driver.find_element_by_css_selector("td.sorting_1").click()
driver.find_element_by_id("sendurlbtn").click()
self.assertEqual("App adaptation", driver.find_element_by_css_selector("h3").text)
self.assertEqual("", driver.find_element_by_css_selector("input.btn.btn-primary").text)
driver.find_element_by_css_selector("input.btn.btn-primary").click()
self.assertEqual("App Composer :: Create/edit an app", driver.title)
self.assertEqual("Details of the jsconfig", driver.find_element_by_css_selector("h5.panel-title").text)
driver.find_element_by_css_selector("input.btn.btn-success").click()
self.assertEqual("App Composer :: Edit the app content", driver.title)
self.assertTrue(self.is_element_present(By.CSS_SELECTOR, "iframe"))
self.assertEqual("Note: Changes will be saved automatically", driver.find_element_by_css_selector("h4.alert.alert-info").text)
self.assertEqual("is a,is part of,has,leads to,influences,increases,decreases", driver.find_element_by_name("relations").get_attribute("value"))
self.assertEqual("Preview", driver.find_element_by_id("preview-tab").text)
driver.find_element_by_id("preview-tab").click()
driver.find_element_by_link_text("Edit").click()
# In PhantomJS we have to wait for this element to be visible.
autoload = driver.find_element_by_name("auto_load")
for i in range(60):
try:
if autoload.is_displayed(): break
except: pass
time.sleep(1)
else: self.fail("time out")
autoload.click()
driver.save_screenshot("seleniumscreenstep.png")
driver.find_element_by_name("relations").click()
driver.find_element_by_name("relations").clear()
driver.find_element_by_name("relations").send_keys("relation1, relation2, relation3")
for i in range(60):
try:
if "All changes saved" == driver.find_element_by_xpath("//body/div[3]").text: break
except: pass
time.sleep(1)
else: self.fail("time out")
driver.find_element_by_link_text("Apps").click()
driver.find_element_by_link_text("Open").click()
for i in range(60):
try:
elem = driver.find_element_by_name("auto_load")
checked = elem.get_attribute("checked")
if checked is None: break
except: pass
time.sleep(1)
else: self.fail("time out")
self.assertEqual("relation1, relation2, relation3", driver.find_element_by_name("relations").get_attribute("value"))
self.assertTrue(self.is_element_present(By.CSS_SELECTOR, "iframe"))
driver.find_element_by_link_text("Apps").click()
driver.find_element_by_link_text("Delete").click()
driver.find_element_by_name("delete").click()
driver.find_element_by_link_text("Log out").click()
except:
driver.save_screenshot("seleniumscreen.png")
raise
def is_element_present(self, how, what):
try: self.driver.find_element(by=how, value=what)
except NoSuchElementException, e: return False
return True
def is_alert_present(self):
try: self.driver.switch_to_alert()
except NoAlertPresentException, e: return False
return True
def close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally: self.accept_next_alert = True
def tearDown(self):
self.driver.quit()
self.assertEqual([], self.verificationErrors)
class AppScreen(unittest.TestCase):
def setUp(self):
reset_database()
if os.environ.get("SELENIUM_HEADLESS"):
self.driver = webdriver.PhantomJS()
else:
self.profile = FirefoxProfile()
self.profile.set_preference("intl.accept_languages", "en")
self.driver = webdriver.Firefox(self.profile)
self.driver.set_window_size(1300, 1000)
self.driver.implicitly_wait(30)
self.base_url = "http://*****:*****@type='submit']").click()
driver.find_element_by_link_text(u"Start composing »").click()
driver.find_element_by_name("name").clear()
driver.find_element_by_name("name").send_keys("dummy1")
driver.find_element_by_css_selector("input.btn.btn-primary").click()
driver.find_element_by_name("saveexit").click()
self.assertEqual("Apps - User Profile", driver.title)
driver.find_element_by_link_text("Home").click()
driver.find_element_by_link_text(u"Start composing »").click()
driver.find_element_by_name("name").clear()
driver.find_element_by_name("name").send_keys("dummy2")
driver.find_element_by_css_selector("input.btn.btn-primary").click()
driver.find_element_by_name("saveexit").click()
self.assertEqual("Apps - User Profile", driver.title)
self.assertEqual("dummy1", driver.find_elements_by_css_selector(".app-title")[1].text)
self.assertEqual("dummy2", driver.find_elements_by_css_selector(".app-title")[0].text)
driver.find_element_by_link_text("Translate").click()
for i in range(60):
try:
if self.is_element_present(By.CSS_SELECTOR, "p.p1"): break
except: pass
time.sleep(1)
else: self.fail("time out")
time.sleep(2)
driver.find_element_by_css_selector("p.p1").click()
driver.find_element_by_id("sendurlbtn").click()
driver.find_element_by_link_text("Apps").click()
self.assertEqual("Concept Mapper", driver.find_element_by_css_selector("div.row > div.row > div.app > div.alert.alert-success > div > h3").text)
self.assertEqual("dummy2", driver.find_elements_by_css_selector(".app-title")[1].text)
self.assertEqual("dummy1", driver.find_elements_by_css_selector(".app-title")[2].text)
self.assertEqual("Composer: translate", driver.find_element_by_css_selector(".app-composer-type").text)
self.assertEqual("Open", driver.find_element_by_link_text("Open").text)
self.assertEqual("Open", driver.find_element_by_xpath("(//a[contains(text(),'Open')])[2]").text)
self.assertEqual("Open", driver.find_element_by_xpath("(//a[contains(text(),'Open')])[3]").text)
driver.find_element_by_id("search_box").send_keys("map")
self.assertEqual("Open", driver.find_element_by_link_text("Open").text)
self.assertFalse(driver.find_element_by_css_selector("div.row > div.row > div.app > div.alert.alert-info > div > h3").is_displayed())
driver.find_element_by_id("search_box").clear()
# MODIFICATION/FIX: In Python empty send_keys does not raise a javascript event. So I put a space and delete
# it.
driver.find_element_by_id("search_box").send_keys(" ")
driver.find_element_by_id("search_box").send_keys(Keys.BACKSPACE)
self.assertEqual("Concept Mapper", driver.find_element_by_css_selector("div.row > div.row > div.app > div.alert.alert-success > div > h3").text)
self.assertEqual("dummy2", driver.find_elements_by_css_selector(".app-title")[1].text)
self.assertEqual("dummy1", driver.find_elements_by_css_selector(".app-title")[2].text)
self.assertEqual("Translate", driver.find_element_by_css_selector("h3").text)
self.assertEqual("Adapt", driver.find_element_by_css_selector("div.alert.alert-info > div > h3").text)
driver.find_element_by_link_text("TRANSLATE").click()
self.assertEqual("App Composer :: Translation tool", driver.title)
driver.find_element_by_link_text("Apps").click()
driver.find_element_by_link_text("ADAPT").click()
self.assertEqual("App Composer :: Choose an App to adapt", driver.title)
driver.find_element_by_link_text("Home").click()
driver.find_element_by_link_text("Apps").click()
self.assertEqual("Apps - User Profile", driver.title)
self.assertEqual("Concept Mapper", driver.find_element_by_css_selector("div.row > div.row > div.app > div.alert.alert-success > div > h3").text)
driver.find_element_by_id("search_box").clear()
driver.find_element_by_id("search_box").send_keys("translate")
self.assertEqual("Concept Mapper", driver.find_element_by_css_selector("div.row > div.row > div.app > div.alert.alert-success > div > h3").text)
driver.find_element_by_link_text("Apps").click()
driver.find_element_by_link_text("Delete").click()
driver.find_element_by_name("delete").click()
driver.find_element_by_xpath("(//a[contains(text(),'Delete')])[2]").click()
driver.find_element_by_name("delete").click()
driver.find_element_by_link_text("Delete").click()
driver.find_element_by_name("delete").click()
driver.find_element_by_link_text("ADAPT").click()
driver.find_element_by_css_selector("td.sorting_1").click()
driver.find_element_by_id("sendurlbtn").click()
driver.find_element_by_css_selector("input.btn.btn-primary").click()
driver.find_element_by_css_selector("input.btn.btn-success").click()
driver.find_element_by_link_text("Apps").click()
driver.find_element_by_link_text("Duplicate").click()
self.assertEqual("Duplication of an application", driver.find_element_by_css_selector("h2").text)
self.assertEqual("Concept Mapper (2)", driver.find_element_by_id("name").get_attribute("value"))
driver.find_element_by_css_selector("button.btn").click()
driver.find_element_by_link_text("Apps").click()
self.assertEqual("Concept Mapper (2)", driver.find_element_by_css_selector("div.row > div.row > div.app > div.alert.alert-info > div > h3").text)
self.assertEqual("Concept Mapper", driver.find_element_by_xpath("//div[2]/div[2]/div/div/h3").text)
driver.find_element_by_link_text("Delete").click()
driver.find_element_by_name("delete").click()
driver.find_element_by_link_text("Delete").click()
driver.find_element_by_name("delete").click()
self.assertEqual("No application.", driver.find_element_by_css_selector("h2").text)
driver.find_element_by_link_text("Log out").click()
except:
driver.save_screenshot("seleniumscreen.png")
raise
def is_element_present(self, how, what):
try: self.driver.find_element(by=how, value=what)
except NoSuchElementException, e: return False
return True
def is_alert_present(self):
try: self.driver.switch_to_alert()
except NoAlertPresentException, e: return False
return True
def close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally: self.accept_next_alert = True
def tearDown(self):
self.driver.quit()
self.assertEqual([], self.verificationErrors)
class Basic(unittest.TestCase):
"""
Mostly generated by Selenium IDE.
Checks that the login/logout procedure flows as expected.
"""
def setUp(self):
if os.environ.get("SELENIUM_HEADLESS"):
self.driver = webdriver.PhantomJS()
else:
self.profile = FirefoxProfile();
self.profile.set_preference("intl.accept_languages", "en")
self.driver = webdriver.Firefox(self.profile)
self.driver.implicitly_wait(30)
self.base_url = "http://*****:*****@type='submit']").click()
try: self.assertEqual("Welcome Administrator!", driver.find_element_by_css_selector("h1").text)
except AssertionError as e: self.verificationErrors.append(str(e))
self.assertEqual("Log out", driver.find_element_by_link_text("Log out").text)
self.assertEqual("Adaptor Composer", driver.find_element_by_xpath("//div[2]/h2").text)
try: self.assertEqual("Translate Composer", driver.find_element_by_xpath("//div[3]/h2").text)
except AssertionError as e: self.verificationErrors.append(str(e))
driver.find_element_by_link_text("Log out").click()
def is_element_present(self, how, what):
try: self.driver.find_element(by=how, value=what)
except NoSuchElementException, e: return False
return True
def is_alert_present(self):
try: self.driver.switch_to_alert()
except NoAlertPresentException, e: return False
return True
def close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally: self.accept_next_alert = True
def tearDown(self):
self.driver.quit()
self.assertEqual([], self.verificationErrors)
def test_that_boolean_prefs_are_written_in_the_correct_format():
profile = FirefoxProfile()
profile.set_preference("sample.bool.preference", True)
profile.update_preferences()
assert profile.default_preferences["sample.bool.preference"] is True
def check_archimedes(check_id, headless, experiment_url, user, password, msg):
"""
This is currently not working on PhantomJS due to its use of an old Qt-related JS core which
does not have function.Prototype.bind.
There may be some workarounds.
:param check_id: Identifier for the check. In this particular check, several sub-checks will be reported.
:param headless: If True the selenium test will use PhantomJS and thus be invisible.
:param experiment_url:
:param user:
:param password:
:param msg: Msg to display for the full check.
:return:
"""
report(TASK_STATE.RUNNING, "%s" % (check_id), msg, "OK")
# Initialize the driver
if headless:
dcap = dict(DesiredCapabilities.PHANTOMJS)
# dcap["phantomjs.page.settings.userAgent"] = (
# "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53 "
# "(KHTML, like Gecko) Chrome/30.0.87"
# )
# dcap["phantomjs.page.customHeaders"] = {
# "Accept-Language": "en-US,en;q=0.5"
# }
driver = webdriver.PhantomJS("phantomjs", desired_capabilities=dcap,
service_args=["--ignore-ssl-errors=true", "--ssl-protocol=tlsv1",
"--web-security=no", "--remote-debugger-port=6500"])
else:
profile = FirefoxProfile()
profile.set_preference("intl.accept_languages", "en")
driver = webdriver.Firefox(profile)
driver.set_window_size(1400, 1000)
driver.implicitly_wait(30)
base_url = experiment_url
try:
# Login
driver.get("""https://weblab.deusto.es/weblab/web/webclient/""")
time.sleep(1)
# Find username & password fields.
usernameField = driver.find_element_by_css_selector("input[type=text]")
usernameField.send_keys(user)
passwordField = driver.find_element_by_css_selector("input[type=password]")
passwordField.send_keys(password)
loginButton = driver.find_element_by_id("login")
if "Log in" not in loginButton.get_attribute("value"): raise CheckException("Log in button not found")
loginButton.click()
time.sleep(2)
# Go to the actual experiment
driver.get(base_url)
print "[We are in the experiment reserve screen]"
archimedes = driver.find_element_by_id("reserve-free-btn")
archimedes.click()
# Reserve can take a long while if there is a queue.
# Wait a while for the frame to load. For now, in seconds.
_wait_by_css(driver, "img.arch-control")
time.sleep(6)
# Find all the up buttons and ensure they are disabled.
buttons = driver.find_elements_by_css_selector("img.arch-control")
# Wait a while before checking
time.sleep(2)
print "Buttons found: %d" % len(buttons)
if len(buttons) != 4 * 7: raise CheckException("Not the expected number of buttons")
# Check that the up buttons are all disabled
for i in range(7):
up = buttons[0 + i * 4]
bn = os.path.basename(up.get_attribute("src"))
if bn != "up.png": raise CheckException("Button is not disabled (basename: %s)" % bn)
# Read the reported weights
initial_weights = read_weights(driver)
print "INITIAL WEIGHTS: %r" % initial_weights
# Lower each ball
for i in range(7):
down = buttons[2 + i * 4]
up = buttons[0 + i * 4]
down.click()
time.sleep(7)
for i in range(7):
down = buttons[2 + i * 4]
up = buttons[0 + i * 4]
# Check that indeed the ball was apparently lowered and raise again.
if os.path.basename(up.get_attribute("src")) != "up_green.png": raise CheckException(
"Up button not enabled after lowering ball (instance %d)" % i)
if os.path.basename(down.get_attribute("src")) != "down.png": raise CheckException(
"Down button not disabled after lowering ball (instance %d)" % i)
up.click()
lowered_weights = read_weights(driver)
print "LOWERED_WEIGHTS: %r" % lowered_weights
# Check whether they were indeed lowered right.
evaluated_weights = []
for i in range(len(lowered_weights)):
w0 = initial_weights[i]
wf = lowered_weights[i]
r = wf / w0
diff = w0 - wf
if diff > 20 or r < 0.8:
# Valid change
evaluated_weights.append(True)
else:
# Something is wrong
evaluated_weights.append(False)
print "EVALUATED WEIGHTS: %r" % evaluated_weights
i = 0
for eval in evaluated_weights:
if eval:
result = "OK"
else:
result = "FAIL"
report(TASK_STATE.FINISHED, "%s>lower-%d" % (check_id, i), "Instance %d: Lowering ball and measuring" % (i),
result)
i += 1
time.sleep(7)
for i in range(7):
down = buttons[2 + i * 4]
up = buttons[0 + i * 4]
# Check that it was re-raised properly.
if os.path.basename(up.get_attribute("src")) != "up.png": raise CheckException(
"Up button not disabled after raising ball again (instance %d)" % i)
if os.path.basename(down.get_attribute("src")) != "down_green.png": raise CheckException(
"Down button not enabled after raising ball again (instance %d)" % i)
time.sleep(2)
# Check whether they were raised right again.
raised_weights = read_weights(driver)
print "RAISED_WEIGHTS: %r" % raised_weights
# Check whether they were indeed lowered right.
evaluated_weights = []
for i in range(len(lowered_weights)):
w0 = lowered_weights[i]
wf = raised_weights[i]
r = w0 / wf
diff = wf - w0
if diff > 20 or r < 0.8:
# Valid change
evaluated_weights.append(True)
else:
# Something is wrong
evaluated_weights.append(False)
print "EVALUATED WEIGHTS (after raising): %r" % evaluated_weights
i = 0
for eval in evaluated_weights:
if eval:
result = "OK"
else:
result = "FAIL"
report(TASK_STATE.FINISHED, "%s>raise-%d" % (check_id, i), "Instance %d: Raising ball and measuring" % (i),
result)
i += 1
print "[ARCHIMEDES]: Everything seems all right"
report(TASK_STATE.FINISHED, "%s" % (check_id), msg, "OK")
except:
driver.save_screenshot("out.png")
print "[ARCHIMEDES]: Thorough archimedes test threw an exception"
report(TASK_STATE.FINISHED, check_id, msg, "FAIL")
raise
driver.close()
def create_download_dir_profile_for_firefox(path_to_download, mime_types_file=None, *extensions_files):
"""
Example use
| ${profile} | Create Download Dir Profile For Firefox | Artifacts | Resources/mimeTypes.rdf | Resources/webdriver_element_locator-2.0-fx.xpi | Resources/selenium_ide-2.9.1-fx.xpi |
| Open Browser Extension | https://support.spatialkey.com/spatialkey-sample-csv-data/ | ff_profile_dir=${profile} |
| Click Element | //a[contains(@href,'sample.csv.zip')] |
"""
path_to_download_check = validate_create_artifacts_dir(path_to_download)
fp = FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.manager.alertOnEXEOpen", False)
fp.set_preference("browser.download.dir", path_to_download_check)
fp.set_preference("xpinstall.signatures.required", False)
fp.set_preference("browser.helperApps.alwaysAsk.force", False)
fp.set_preference("browser.helperApps.neverAsk.saveToDisk",
"application/msword;application/csv;text/csv;image/png;image/jpeg;application/pdf;text/html;text/plain;application/octet-stream")
fp.set_preference("pdfjs.disabled", True)
fp.update_preferences()
for single_extension in extensions_files:
fp.add_extension(single_extension)
if mime_types_file is not None:
from shutil import copy2
copy2(os.path.normpath(mime_types_file), fp.profile_dir)
logger.info("Firefox Profile Created in dir '" + fp.profile_dir + "'")
return fp.profile_dir
class ApplicationIndexer(object):
def __init__(self, url, retries, acknowledgements):
self.url = url
self.attempt = 0
self.retries = retries
self.acknowledgements = acknowledgements
self.proxy_test = google_prop.proxy_test
self.fp = FirefoxProfile()
#self.fp.set_preference('permissions.default.stylesheet', 2) # Disable css
self.fp.set_preference('permissions.default.image', 2) # Disable images
self.fp.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false') # Disable Flash
self.fp.set_preference('general.useragent_rotator.override', useragent.get_random_agent(google_prop.user_agent_list_url))
self.fp.set_preference("network.proxy.type", 1)
self.proxy_details = proxy.get_random_proxy(google_prop.proxy_list_url)
self.proxy = Proxy({
'httpProxy': self.proxy_details,
'sslProxy': self.proxy_details
})
pass
def get_scraped_apps(self, scroll_script):
applications = self.get_applications_in_page(scroll_script)
return applications
pass
def get_applications_in_page(self, scroll_script):
applications = []
driver = None
try:
desired_capabilities = dict(DesiredCapabilities.PHANTOMJS)
desired_capabilities["phantomjs.page.settings.userAgent"] = useragent.get_random_agent(google_prop.user_agent_list_url)
service_args = ['--load-images=no', '--proxy=%s' % (proxy.get_random_proxy(google_prop.proxy_list_url))]
driver = PhantomJS(desired_capabilities=desired_capabilities, service_args=service_args)
# driver = Firefox(firefox_profile=self.fp, proxy=self.proxy)
if self.proxy_test:
driver.get('http://curlmyip.com/')
ip = driver.find_element_by_xpath('//body//pre').text
print('ip : [ ' + ip + ' ]')
pass
else:
driver.get(self.url)
driver.execute_script(scroll_script)
acknowledge = 0
done = False
while not done:
scroll_finished = driver.execute_script("return scraperLoadCompleted")
if scroll_finished:
if acknowledge == self.acknowledgements:
done = driver.execute_script("return scraperLoadCompleted")
pass
else:
acknowledge += 1
pass
pass
else:
acknowledge = 0
pass
time.sleep(5) # Wait before retry
pass
product_matrix = driver.find_elements_by_class_name("card")
for application in product_matrix:
extracted_application = self.extract_application_data(application)
# if extracted_application['app_price'] != -1:
applications.append(extracted_application)
#pass
pass
pass
driver.quit()
pass
except Exception as e:
if driver is not None:
driver.quit()
pass
if self.attempt < self.retries:
self.attempt += 1
time.sleep(10)
print 'retry : url [ ' + self.url + ' ] + | attempt [ ' + str(self.attempt) + ' ] | error [ ' + str(e) + ' ]'
applications = self.get_applications_in_page(scroll_script)
pass
else:
print('fail : url [ ' + self.url + ' ] | error [ ' + str(e) + ' ]')
pass
pass
return applications
pass
@staticmethod
def extract_application_data(application):
app_id = application.get_attribute("data-docid") # ID of the application
card_content = application.find_element_by_class_name("card-content")
app_url = card_content.find_element_by_xpath("a").get_attribute("href") # URL of the application
price_containers = card_content.find_elements_by_class_name('buy')
author_url = application.find_element_by_class_name("subtitle").get_attribute("href")
app_price = 0.0
for price_container in price_containers:
if '$' in price_container.text.lower():
try:
app_price_string = price_container.text.replace('$', '')
app_price = float(app_price_string)
break
pass
except Exception:
pass
pass
elif 'free' in price_container.text.lower():
app_price = -1
break
pass
pass
extracted_data = {
'app_id': app_id,
'app_url': app_url,
'app_price': app_price,
'author_url': author_url
}
return extracted_data
pass
pass
class SeleniumBaseTest(unittest.TestCase):
def start_test_weblab_deployment(self):
# Load the configuration of the weblab instance that we have set up for just this test.
self.global_config = load_dir('test/deployments/webclient_dummy')
# Start the weblab instance. Because we have activated the dont-start flag it won't actually
# start listening on the port, but let us use the Flask test methods instead.
self.handler = self.global_config.load_process('myhost', 'myprocess')
self.core_server = GLOBAL_REGISTRY['mycore:myprocess@myhost']
@classmethod
def setUpClass(self):
# Starts a test deployment.
self.start_test_weblab_deployment()
if os.environ.get("SELENIUM_HEADLESS") and False:
self.driver = webdriver.PhantomJS()
else:
self.profile = FirefoxProfile()
self.profile.set_preference("intl.accept_languages", "en")
self.driver = webdriver.Firefox(self.profile)
self.driver.set_window_size(1400, 1000)
# self.driver.implicitly_wait(30)
self.base_url = "http://localhost:5000/"
self.verificationErrors = []
self.accept_next_alert = True
def is_element_present(self, how, what):
try: self.driver.find_element(by=how, value=what)
except NoSuchElementException, e: return False
return True
def is_alert_present(self):
try: self.driver.switch_to_alert()
except NoAlertPresentException, e: return False
return True
def close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally: self.accept_next_alert = True
@classmethod
def tearDownClass(self):
self.driver.quit()
self.assertEqual([], self.verificationErrors)
