def test_safari():
'''
A selenium driver exists for Safari, but it's not headless as far as I can tell
'''
from selenium.webdriver import Safari
print(dir(Safari), '\n')
from selenium.webdriver import safari
print(dir(safari))
browser = Safari()
browser.get('https://duckduckgo.com')
def safari_driver(request: "SubRequest") -> Union[Remote, Safari]:
"""Fixture for receiving selenium controlled Safari instance"""
if request.cls.test_type == "safari-local":
driver = Safari()
else:
executor = RemoteConnection(SAUCE_HUB_URL, resolve_ip=False)
driver = Remote(desired_capabilities=SAUCE_SAFARI,
command_executor=executor)
set_selenium_driver_timeouts(driver)
request.cls.driver = driver
yield driver
driver.close()
def get_browser_driver():
"""
Using following env vars:
WEB_DRIVER: name of the driver "chrome", "firefox"
Get web-driver - choose between Chrome, Firefox etc.
:return: driver instance object
"""
capabilities = dict(getattr(DesiredCapabilities, WEB_DRIVER))
log.info(f"Webdriver is: {WEB_DRIVER}, running headless: {HEADLESS}")
if WEB_DRIVER == 'CHROME':
chromedriver = os.path.join(PROJECT_PATH, "chromedriver.exe")
chrome_options = ch_options()
if HEADLESS:
chrome_options.set_headless()
browser_driver = Chrome(executable_path=chromedriver,
desired_capabilities=capabilities,
options=chrome_options)
elif WEB_DRIVER == 'FIREFOX':
geckodriver = os.path.join(PROJECT_PATH, "geckodriver.exe")
firefox_options = ff_options()
if HEADLESS:
firefox_options.set_headless()
browser_driver = Firefox(executable_path=geckodriver,
capabilities=capabilities,
options=firefox_options)
elif WEB_DRIVER == 'SAFARI':
safaridriver = os.path.join(PROJECT_PATH, "safaridriver.exe")
# headless mode is not possible right now in Safari
browser_driver = Safari(executable_path=safaridriver,
capabilities=capabilities)
else:
raise Exception('Unknown/unsupported driver selected: ' + WEB_DRIVER)
return browser_driver
def _create_safari(cls, config, driver_path, browser_bin_path):
from selenium.webdriver import Safari
caps = DesiredCapabilities.SAFARI
caps.update(config["driverCapabilities"])
return Safari(executable_path=driver_path, desired_capabilities=caps)
def _set_selenium_driver(self, driver_type, firefox_profile):
if driver_type == self.DriverType.CHROME:
self._web_driver = Chrome()
elif driver_type == self.DriverType.FIREFOX:
if firefox_profile and os.path.exists(firefox_profile):
profile = FirefoxProfile(firefox_profile)
self._web_driver = Firefox(firefox_profile=profile)
else:
self._web_driver = Firefox()
elif driver_type == self.DriverType.IE:
self._web_driver = Ie()
elif driver_type == self.DriverType.SAFARI:
self._web_driver = Safari()
elif driver_type == self.DriverType.CHROME_HEADLESS:
profile = ChromeOptions()
profile.add_argument('headless')
profile.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
self._web_driver = Chrome(options=profile)
else:
self._web_driver = Chrome()
print("Invalid Driver Type filled: %r" % driver_type)
def using_safari() -> "BrowseTheWeb":
"""
Creates and uses a default Safari Selenium webdriver instance. Use
this if you don't need to set anything up for your test browser.
Returns:
|BrowseTheWeb|
"""
return BrowseTheWeb.using(Safari())
def mvideo_hits_collector():
driver = Safari()
driver.maximize_window()
db = client['Goods_from_hits_mvideo']
goods_in_db = db.goods_in_db
driver.get('https://www.mvideo.ru')
gallery_titles = driver.find_elements_by_xpath(
"//div[@class='gallery-title-wrapper']")
for gal_title in gallery_titles:
if 'Хиты продаж' in gal_title.text:
hits_block = gal_title.find_element_by_xpath(
"./ancestor::div[@class='section']")
next_button = hits_block.find_element_by_xpath(
".//a[contains(@class, 'next-btn')]")
while True:
next_button.click()
WebDriverWait(hits_block, 10).until(
EC.element_to_be_clickable((By.XPATH, ".//li//h4/a")))
WebDriverWait(hits_block, 10).until(
EC.element_to_be_clickable(
(By.XPATH, ".//a[contains(@class, 'next-btn')]")))
if 'disable' in next_button.get_attribute('class'):
break
goods_list_html = hits_block.find_elements_by_xpath(".//li//h4/a")
for good in goods_list_html:
good_dict = {}
good_url = good.get_attribute('href')
good_desc = json.loads(good.get_attribute('data-product-info'))
good_name = good_desc['productName']
good_price = float(good_desc['productPriceLocal'])
good_category = good_desc['productCategoryName']
good_dict['url'] = good_url
good_dict['name'] = good_name
good_dict['price'] = good_price
good_dict['category'] = good_category
goods_in_db.update_one({'url': good_url}, {'$set': good_dict},
upsert=True)
driver.close()
def __init__(self, usr, browser=None, headless=False):
self.BASE_URL = "https://www.instagram.com/"
if browser == "safari":
# TODO
# Not optimized yet
self.browser = Safari()
elif browser == "firefox":
# TODO
# Not optimized yet
self.browser = Firefox()
else: # Chrome
if not os.path.exists("path_chromedriver.txt"):
raise IOError("Path to chromedriver is missing.")
# Your path to chromedriver
with open("path_chromedriver.txt", "r") as f:
path_chromedriver = f.readline()
f.close()
# Open in headless mode
if headless is True:
self.op = ChromeOptions()
self.op.add_argument('headless')
self.browser = Chrome(executable_path=path_chromedriver,
options=self.op)
else:
self.browser = Chrome(executable_path=path_chromedriver)
self.browser.get(self.BASE_URL + usr)
self.status = get(self.BASE_URL + usr).status_code
self.logged_in = False
def _open_browser(self, browser_name):
"""
:return: webdriver
"""
if browser_name == 'chrome':
if self.options is not None:
self.driver = Chrome(chrome_options=self.options)
else:
self.driver = Chrome()
elif browser_name == 'ie':
self.driver = Ie()
elif browser_name == 'safari':
self.driver = Safari()
elif browser_name == 'edge':
self.driver = Edge()
elif browser_name == 'firefox':
self.driver = Firefox()
else:
raise Exception('Faild input browser name')
self.driver.get(self.base_url)
return self.driver
def driver(request, logger, proxy):
browser = request.config.getoption("--browser")
implicit_wait = request.config.getoption("--wait")
if browser == 'chrome':
chrome_options = Options()
chrome_options.add_argument(f'--proxy-server={proxy.proxy}')
chrome_options.add_experimental_option('w3c', False)
# chrome_options.add_argument("--start-fullscreen")
# chrome_options.add_argument("--headless")
driver = EventFiringWebDriver(Chrome(options=chrome_options), MyListener(logger))
# driver = webdriver.Remote(
# "http://localhost:4444/wd/hub",
# desired_capabilities={'browserName': 'chrome'})
elif browser == 'firefox':
driver = Firefox()
elif browser == 'safari':
driver = Safari()
else:
raise ValueError(f'Unrecognized browser {browser}')
driver.implicitly_wait(implicit_wait)
yield driver
driver.quit()
def mailru_collector():
driver = Safari()
driver.maximize_window()
db = client['Sends_from_mailru']
sends_in_db = db.sends_in_db
driver.get('https://mail.ru')
elem = driver.find_element_by_id('mailbox:login-input')
elem.send_keys(get_login())
elem.submit()
elem = WebDriverWait(driver, 10).until(
EC.visibility_of(driver.find_element_by_id('mailbox:password-input')))
elem.send_keys(get_password())
elem.submit()
WebDriverWait(driver, 10).until(EC.title_contains('- Почта Mail.ru'))
sends_block = driver.find_element_by_xpath(
"//div[@class='dataset__items']")
sends = set()
while True:
len_of_sends_list = len(sends)
list_sends = sends_block.find_elements_by_xpath(".//a[@href]")
for send in list_sends:
sends.add(send.get_attribute('href'))
if len(sends) == len_of_sends_list:
break
var = list_sends[-1].location_once_scrolled_into_view
time.sleep(1)
while sends:
send_dict = {}
send_url = sends.pop()
driver.get(send_url)
WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.XPATH, "//div[@class='letter-body']")))
text_send = driver.find_element_by_xpath(
"//div[contains(@class, 'letter-body')]").text
title_send = driver.find_element_by_tag_name('h2').text
from_send = driver.find_element_by_class_name('letter-contact').text
date_send = driver.find_element_by_class_name('letter__date').text
send_dict['text'] = text_send
send_dict['title'] = title_send
send_dict['from'] = from_send
send_dict['date'] = date_send
send_dict['url'] = send_url
sends_in_db.update_one({'url': send_url}, {'$set': send_dict},
upsert=True)
driver.close()
ans.send_keys(emailId)
elif (question.text.rstrip())=="CGPA":
print("answering",question.text )
ans=person.find_element_by_xpath('.//input[@class="quantumWizTextinputPaperinputInput exportInput"]')
ans.send_keys(currentCgpa)
else:
print("answer not found for", question.text)
continue
driver = Safari()
driver.get("https://docs.google.com/forms/d/e/1FAIpQLSc5NC4HJX6hf8QqLhixRLwtGMisa4MYVmLV5ixPqYUIYJNHnw/viewform?usp=sf_link")
driver.implicitly_wait(7000)
def checkform():
try:
assert "https://docs.google.com/forms/" in driver.current_url
return True
except:
print("match not found")
return False
from selenium.webdriver import Safari
b = Safari()
b.get('http://google.com')
e = b.find_element_by_id('lst-ib')
e.click() # is optional, but makes sure the focus is on editbox.
e.send_keys('12.34')
e.get_attribute('value')
# outputs: u'12.34'
e.click()
e.clear()
e.get_attribute('value')
# outputs: u''
e.send_keys('56.78')
e.get_attribute('value')
# outputs: u'56.78'
# importing selenium libararies
from selenium.webdriver import Safari
#import selenium libraray to amke keys working through
#automation
from selenium.webdriver.common.keys import Keys
# calling the web driver of safari
driver = Safari()
#taking user id and passowrd input from the user
# you can hard code this if you want to save
# this file locallly on your computer
user1 = input("enter your user id for moddle :")
passw1 = input("enter your password :"******"https://moodle.iitd.ac.in/login/index.php")
# finding the user id and password elements on the login page
user = driver.find_element_by_id("username")
passw = driver.find_element_by_id("password")
# entering the user id and pasword in the form
user.send_keys(user1)
passw.send_keys(passw1)
# finding the capthca input in the login page
captcha = driver.find_element_by_id("valuepkg3")
# making a local copy of the login form so that
# captcha can be extracted and solved
text1 = driver.find_element_by_id("login").text
# figuring out which type of capthca is asked
case1 = text1[326:329] # for add or sub
case2 = text1[332:335] # for first or second value
def checking_the_type_of_captha(case1, case2):
from selenium.webdriver import Safari
# importing os module to making folders
# your PC
import os
driver = Safari()
#taking input from user for the problem they want to scrape
problem = int(input("please enter the problem you want to scrape:"))
#creating the folder name with problem
os.mkdir("/Users/lakshaydagar/Downloads/%s" % (problem))
# navigating to the codeforces site
driver.get("https://codeforces.com/contest/%s" % (problem))
# finding the name of variable of part of problem
taga = driver.find_elements_by_tag_name("a")
#runnig a loop through all the parts in problem
for i in range(27, len(taga) - 9, 4):
driver.get("https://codeforces.com/contest/%s" % (problem))
taga = driver.find_elements_by_tag_name("a")
# converting the part of problem to a string
op = taga[i].get_attribute("innerHTML")
o = op[29] # mapping is done such that the 29th element in page
# source is first part of problem and folllwoing part follow
# a 4 no. differance
try:
# navigating to specific problem page
driver.get("https://codeforces.com/contest/%s/problem/%s" %
(problem, o))
# creating a sprate folder for the part of the problem
os.mkdir("/Users/lakshaydagar/Downloads/%s/%s" % (problem, o))
# maximizing the window so that screenshot can be taken
driver.set_window_size(1500, 1080)
# saving the screenshot in the problem folder created earlier
def browser():
browser = Safari()
return browser
"""this code will scrape the subreddit 'WallStreetBets' and will return a text file containing the most mentioned tickers on the subreddit."""
from datetime import date,timedelta
from dateutil.parser import parse
import requests
from selenium.webdriver import Safari
import csv
# Setting up date stuff
today = date.today()
yesterday = today - timedelta(days=1)
day_index = today.weekday()
with Safari() as driver:
Weekday_URL = "https://www.reddit.com/r/wallstreetbets/search/?q=%22Daily%20Discussion%20Thread%22%20flair%3A%22Daily%20Discussion%22&restrict_sr=1&sort=new"
Weekend_URL = "https://www.reddit.com/r/wallstreetbets/search/?q=%22Weekend%20Discussion%22%20flair%3A%22Weekend%20Discussion%22&restrict_sr=1&sort=new"
if day_index in {1,2,3,4,5}:
driver.get(Weekday_URL)
elif day_index in {0,6}:
driver.get(Weekend_URL)
links = driver.find_elements_by_xpath('//*[@class="_eYtD2XCVieq6emjKBH3m"]')
for a in links:
if a.text.startswith('Daily Discussion Thread'):
date = "".join(a.text.split(' ')[-3:])
parsed = parse(date)
"""iteratate through all the existing csv sheets, make apprioriate HTML files, take screenerz"""
from os import listdir, system
from os.path import isfile, join
import re
from selenium.webdriver import Safari
browser = Safari()
PATH = './data/xls/out/'
def take_screen(html_add):
p = re.compile(r'(\d{6})')
match = re.findall(p, html_add)
LOCAL_HOST = 'http://127.0.0.1:8080/html/'
browser.get('http://127.0.0.1:8080/' + html_add)
browser.get_screenshot_as_file('./screen/{0}'.format(match[0]) + '.png')
def make_html(doc):
# Read in the file
# with open('{0}{1}'.format(PATH, doc), 'r') as file:
with open('base.html', 'r') as file:
filedata = file.read()
# Replace the target string
filedata = filedata.replace('CSVFILENAMETOCHANGE',
'../' + PATH + doc.split('.')[0])
# Write the file out again
html_file = './html/{0}.html'.format(doc.split('.')[0])
from BorsaReader.Extractor_Nasdaq import produce_urls_nasdaq, get_tables_nasdaq
import pandas as pd
from selenium.webdriver import Safari
save_path = '/Users/albertoferrando/Google Drive/Progetti/Investimenti/BorsaReader/Storage/listing_nyse.csv'
base = 'http://eoddata.com/stocklist/NYSE/'
urls = produce_urls_nasdaq(base)
tables = []
driver = Safari()
for url in urls:
print(url)
tables.append(get_tables_nasdaq(url, driver))
tables = pd.concat(tables)
tickers = tables['Code'].to_frame()
tickers.column = 'Ticker'
tickers.to_csv(save_path, index=False)
driver.quit()
from selenium.webdriver import Chrome, Edge, Firefox, Ie, PhantomJS, Safari
ChromeDriver = lambda path, *args, **kwargs: lambda: Chrome(
path, *args, **kwargs)
EdgeDriver = lambda path, *args, **kwargs: lambda: Edge(path, *args, **kwargs)
FirefoxDriver = lambda path, *args, **kwargs: lambda: Firefox(
*args, executable_path=path, **kwargs)
IeDriver = lambda path, *args, **kwargs: lambda: Ie(path, *args, **kwargs)
PhantomJSDriver = lambda path, *args, **kwargs: lambda: PhantomJS(
path, *args, **kwargs)
SafariDriver = lambda path, *args, **kwargs: lambda: Safari(
path, *args, **kwargs)
