def get_webdriver_options(cls):
kwargs = super(FirefoxBase, cls).get_webdriver_options()
if cls.firefox_binary is not None:
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
kwargs['firefox_binary'] = FirefoxBinary(firefox_path=cls.firefox_binary)
return kwargs
def main(args) :
statusMode = args.status
mat = args.mat
pwd = args.pwd
driverTarget = args.driverTarget
autoMode = args.auto
criticalTime = args.criticalTime
binaryLoc = args.binaryLoc
if not(statusMode) :
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
url = "http://consulta.uffs.edu.br/pergamum/biblioteca_s/php/login_usu.php" #URL de acesso ao pergamum
print("Iniciando sessão do requests...")
with requests.Session() as s:
if not(statusMode) :
print("Iniciando Selenium Webdriver...")
if (driverTarget=='chromedriver') :
options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument('--log-level=3')
driver=webdriver.Chrome(binaryLoc,options=options)
if (driverTarget=='firefox_binary') :
options = webdriver.FirefoxOptions()
options.add_argument('-headless')
binary = FirefoxBinary(binaryLoc)
driver = webdriver.Firefox(firefox_binary=binary,options=options)
if (driverTarget=="geckodriver") :
options = webdriver.FirefoxOptions()
options.add_argument('-headless')
driver = webdriver.Firefox(executable_path=r"".join(binaryLoc),options=options)
driver.get(url)
request_cookies_selenium = driver.get_cookies()
for c in request_cookies_selenium : s.cookies.set(c['name'], c['value'])
#Dados para o request
if (mat=='' or pwd=='') : print("\nInsira os dados para login no pergamum...")
if (mat=='') : mat=input("Matricula: ")
if (pwd=='') : pwd=getpass("Senha: ")
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36',
'Referer': url,
'Host': 'consulta.uffs.edu.br',
'Connection': 'keep-alive'
}
auth_data = {
'flag': 'index.php',
'login': mat,
'password': pwd,
'button': 'Access',
'numero_mestre': '',
'ifsp_categ': '',
'lab_com_solicitacao': ''
}
print()
#Request
s.get(url, headers=headers)
web_r = s.post(url, data=auth_data, headers=headers)
logged_url=web_r.url
#Teste de conexão
if (logged_url.find('meu_pergamum')==-1): print("Erro de login"); exit(0) ##
print("Login realizado com sucesso!\nUsuário: ",end='')
web_soup=BeautifulSoup(web_r.content, 'lxml')
u= str(web_soup.find(attrs={"id": "nome"}))
print(u[re.compile(r'<strong>').search(u).span()[1]:re.compile(r'</strong>').search(u).span()[0]])
print()
#Extração dos dados dos livros
books_div = str(web_soup.find(attrs={"class": "c1"}))
#print(books_div)
#Extração dos nomes dos livros
print("Livros pendentes:")
books_names = re.findall(r'title=".*"',books_div)
if (books_names==[]) : print("NENHUM"); exit(0) ##
else :
for i in range(len(books_names)) :
books_names[i]=books_names[i].replace('<b>','')
books_names[i]=books_names[i].replace('</b>','')
books_names[i]=re.sub(r'(\s"|title=")','', books_names[i])
books_names[i]=re.sub(r'\s\s\s',' ', books_names[i])
books_names[i]=re.sub(r'\s\s',' ', books_names[i])
#Extração dos dias de expiração
needs_renew=[False]*len(books_names)
books_exp = re.findall(r'\d\d/\d\d/\d\d\d\d',books_div)
for i in range(len(books_exp)) :
if ((datetime.datetime.now() + datetime.timedelta(days=criticalTime)) >= (datetime.datetime.strptime(books_exp[i],'%d/%m/%Y'))) : needs_renew[i]=True
#Print dos dados
t = PrettyTable(['Nome', 'Data de expiraçação'])
for i in range(len(books_names)) :
t.add_row([books_names[i], books_exp[i]+(' *' if needs_renew[i] else '')])
print(t)
print("\"*\" = Precisa de atenção para ser renovado")
if (statusMode) : exit()
print()
#Confirmação de quais renovar
if not(autoMode) :
print("Quais livros deseja renovar? (0=nenhum) ",end='')
books_to_renew=list(input().split(','))
print()
if len(books_to_renew)==0 or books_to_renew.count('0')>0 : print("Nada para renovar, saindo..."); exit(0) ##
for b in books_to_renew.copy() :
if b.find('-')!=-1 :
for i in range(int(b[0])-1,int(b[2])) :
books_to_renew.append(i)
else :
books_to_renew.append(int(b)-1)
del books_to_renew[books_to_renew.index(b)]
books_to_renew=list(set(books_to_renew)); books_to_renew.sort()
else :
books_to_renew=[i for i in range(len(needs_renew)) if needs_renew[i]]
if len(books_to_renew)==0 : print("Nada para renovar, saindo..."); exit(0) ##
print("Iniciando processo de renovação...")
print("Injetando cookies no selenium...\n")
#Chamada do selenium
dict_webr_cookies = web_r.cookies.get_dict()
response_cookies_selenium = [{'name':name, 'value':value} for name, value in dict_webr_cookies.items()]
for c in response_cookies_selenium : driver.add_cookie(c)
driver.get(logged_url)
print("Começando a renovar livro a livro...\n")
#Chamada de renovar
for i in books_to_renew :
print("Renovando livro: %s... "%books_names[i])
renovarBtns = driver.find_elements_by_class_name("btn_renovar")
renovarBtns[i].send_keys(Keys.RETURN)
time.sleep(3)
#if (driver.current_url.find('erro')!=-1) : print("ERRO...")
#else : print("Sucesso...")
driver.execute_script("window.history.go(-1)")
time.sleep(3)
print("\nFim...")
driver.quit()
def setUp(self):
binary = FirefoxBinary(r'C:\Program Files\Mozilla Firefox\firefox.exe')
self.browser = webdriver.Firefox(firefox_binary=binary)
#!/usr/bin/env python
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
import time
import threading
capabilities = webdriver.DesiredCapabilities().FIREFOX
capabilities["marionette"] = False
binary = FirefoxBinary(r'/usr/bin/firefox')
class multiClient():
def __init__(self):
self.hostname = "http://localhost"
self.port = 9000
self.gamename = "game2"
self.teams = []
self.usernames = []
self.emails = []
self.teamname1 = "hard"
self.teamname2 = "easy"
def newBrowserWindow(self):
return webdriver.Firefox(firefox_binary=binary)
def openApplication(self, driver=None):
if (not (driver)):
self.driver.get(self.hostname + ":" + str(self.port))
else:
try:
login(driver)
for i in tasklist:
logger.info('fill date: %s' % i['taskdiary']['date'])
select_task(driver, i['taskid'])
fill_tasklog(driver, i['taskdiary'])
except Exception as e:
logger.error(e)
finally:
driver.quit()
if __name__ == '__main__':
url = r'http://rdmprd.fiberhome.com.cn/main.do'
firefoxdriver = r'D:\Program Files\Mozilla Firefox\firefox.exe'
firebin = FirefoxBinary(firefoxdriver)
jsfilepath = os.path.join(
os.path.split(os.path.realpath(__file__))[0], r'rdmtask.json')
with open(jsfilepath, 'r', encoding='utf-8') as fp:
data = json.load(fp)
tasknum = len(data)
tasklist = []
for i in range(tasknum):
for j in range(len(data[i]['taskdiary'])):
tasklist.append({
'taskid': data[i]['taskid'],
'taskdiary': data[i]['taskdiary'][j]
})
process(tasklist)
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
binary = FirefoxBinary('path/to/installed firefox binary')
browser = webdriver.Firefox(firefox_binary=binary)
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
binary = FirefoxBinary('/usr/local/bin/geckodriver')
driver = webdriver.Firefox(firefox_binary=binary)
url = "https://www.instagram.com/explore/locations/1006877751/?__a=1&max_id="
geturl = driver.get(url)
print geturl
def test_set_binary_with_firefox_binary(options):
binary = FirefoxBinary('foo')
options.binary = binary
assert options._binary == binary
def test_get_binary_location(options):
options._binary = FirefoxBinary('/foo')
assert options.binary_location == '/foo'
# loading required packages
import pandas as pd
import os
import time
import datetime
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from bs4 import BeautifulSoup
# initializing needed variables
realtor_url = 'https://www.realtor.ca/on/toronto/real-estate-for-sale'
click_next_button = "/html/body/form/div[5]/div[2]/span/div/div[4]/div[3]/span/span/div/a[3]"
# launching forefox webdriver
binary = FirefoxBinary(
'C:/Users/Prince.Atul/AppData/Local/Mozilla Firefox/firefox')
driver = webdriver.Firefox(firefox_binary=binary)
# saving all the relevant pages of the website
def scrapping_function(current_url, click_next_button):
driver.get(current_url)
i = 1
next_page_true = True
while next_page_true:
page = driver.page_source
file_ = open('saved_pages/toronto/page_{}.html'.format(i), 'w')
file_.write(page)
file_.close()
page_url = driver.current_url
elem = driver.find_element_by_xpath(click_next_button)
def run_tests(firefox_path=None):
basedir = os.path.dirname(__file__)
driver = None
profile = FirefoxProfile()
if firefox_path:
if sys.platform == "darwin" and os.path.isdir(firefox_path):
firefox_path = os.path.join(firefox_path, "Contents", "MacOS",
"firefox")
binary = FirefoxBinary(firefox_path)
else:
binary = None
try:
build1 = tempfile.NamedTemporaryFile(mode="wb",
suffix=".xpi",
delete=False)
build2 = tempfile.NamedTemporaryFile(mode="wb",
suffix=".xpi",
delete=False)
try:
createBuild(basedir, type="gecko", outFile=build1)
createBuild(os.path.join(basedir, "testhelper"),
type="gecko",
outFile=build2)
profile.add_extension(build1.name)
profile.add_extension(build2.name)
finally:
os.unlink(build1.name)
os.unlink(build2.name)
driver = WebDriver(profile, firefox_binary=binary)
driver.wait_until = lambda method: WebDriverWait(
driver, default_timeout).until(lambda d: method())
driver.accept_alert = Alert(driver).accept
driver.keys = Keys
def chain(*actions):
for action in actions:
c = ActionChains(driver)
action(c)
c.perform()
driver.chain = chain
max_timestamp = {"value": 0}
def get_urls():
result = []
prefix = "[testhelper] Loading: "
new_timestamp = max_timestamp["value"]
for item in driver.get_log("browser"):
timestamp = item["timestamp"]
if timestamp <= max_timestamp["value"] or not item[
"message"].startswith(prefix):
continue
if timestamp > new_timestamp:
new_timestamp = timestamp
result.append(item["message"][len(prefix):])
max_timestamp["value"] = new_timestamp
return result
driver.get_urls = get_urls
def close_background_tabs():
driver.execute_script('''
var event = document.createEvent("Events");
event.initEvent("testhelper_closeBackgroundTabs", true, false);
document.dispatchEvent(event);
''')
driver.close_background_tabs = close_background_tabs
def middle_click(self):
driver.execute_script(
'''
var event = document.createEvent("Events");
event.initEvent("testhelper_middleclick", true, false);
arguments[0].dispatchEvent(event);
''', self)
WebElement.middle_click = middle_click
environment = {
"__builtins__": __builtins__,
"driver": driver,
}
testdir = os.path.join(basedir, "tests")
for filename in os.listdir(testdir):
if filename.startswith(".") or not filename.endswith(".py"):
continue
filepath = os.path.join(testdir, filename)
environment["__file__"] = filepath
with open(filepath, "rb") as handle:
exec handle in environment
finally:
if driver:
driver.quit()
shutil.rmtree(profile.path, ignore_errors=True)
except WebDriverException as e:
error = e.__str__()
if "executable needs to be in PATH" in e.__str__():
print("ChromeDriver isn't installed. Check test/chromium/README.md " \
"for instructions on how to install ChromeDriver")
sys.exit(2)
else:
raise e
if sys.argv[1] == "Firefox":
fp = webdriver.FirefoxProfile(sys.argv[2])
try:
if len(sys.argv) == 4:
binary = FirefoxBinary(sys.argv[3])
driver = webdriver.Firefox(fp,
firefox_binary=binary,
log_path=os.devnull)
else:
driver = webdriver.Firefox(fp, log_path=os.devnull)
except WebDriverException as e:
error = e.__str__()
if "executable needs to be in PATH" in e.__str__():
print("GeckoDriver isn't installed. Check test/firefox/README.md " \
"for instructions on how to install GeckoDriver")
sys.exit(2)
else:
raise e
from bs4 import BeautifulSoup
import requests
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import time
from selenium.webdriver.common.keys import Keys
import datetime as dt
# firefox를 통해서 크롤링
binary = FirefoxBinary('C:\\Program Files\\Mozilla Firefox\\firefox.exe')
browser = webdriver.Firefox(
executable_path='D:\\inu\\Database Project\\geckodriver.exe',
firefox_binary=binary)
# 시간
startdate = dt.date(year=2018, month=9, day=1)
untildate = dt.date(year=2018, month=9, day=2)
enddate = dt.date(year=2018, month=11, day=5)
# 파일 생성
f = open(".\\log.txt", 'w')
totalfreq = []
while not enddate == startdate:
url = 'https://twitter.com/search?q=서브웨이레시피%20since%3A' + str(
startdate) + '%20until%3A' + str(
untildate) + '&amp;amp;amp;amp;amp;lang=eg'
browser.get(url)
html = browser.page_source
soup = BeautifulSoup(html, 'html.parser')
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.common.exceptions import NoSuchElementException
import pandas as pd
import xlrd, xlwt
from LK.models import Air
# интеграция с Гугл таблицами
import gspread
from oauth2client.service_account import ServiceAccountCredentials
import pprint, re
# интеграция с Гугл таблицами
capabilities = webdriver.DesiredCapabilities().FIREFOX
capabilities["marionette"] = True
binary = FirefoxBinary('/root/FireFox/firefox/firefox')
driver = webdriver.Firefox(firefox_binary=binary)
listClient = []
listPhoneNumber = []
listClientAddress = []
rb = xlrd.open_workbook('/root/Yandex.Disk/самолеты/ГЖ/Клиент-телефон-НП.xlsx',
formatting_info=False)
sheet = rb.sheet_by_index(0)
for rownum in range(sheet.nrows):
row = sheet.row_values(rownum)
listClient.append(row[0])
listPhoneNumber.append(row[1])
listClientAddress.append(row[5])
# listClientAddress
dictionary = {
#importing required libraries
from bs4 import BeautifulSoup
import pandas as pd
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
#giving path to driver for accessing the browser
binary = FirefoxBinary('/usr/lib/firefox/firefox')
driver = webdriver.Firefox(firefox_binary=binary)
#executing driver to access the link
driver.get("https://www.flipkart.com/search?q=mobile+phone&as=on&as-show=on&otracker=AS_Query_OrganicAutoSuggest_1_6&otracker1=AS_Query_OrganicAutoSuggest_1_6&as-pos=1&as-type=RECENT&as-backfill=on")
#creating empty list to store the data
names = []
prices = []
ratings = []
#getting the content of webpage
content = driver.page_source
soup = BeautifulSoup(content,"lxml")
#finding the data with specified tag
for i in soup.find_all('a' , href=True ,attrs={'class':'_31qSD5'}):
name = i.find('div' , attrs={'class':'_3wU53n'})
price = i.find('div' , attrs={'class' : '_1vC4OE _2rQ-NK'})
rating = i.find('div' , attrs = {'class': 'hGSR34'})
names.append(name.text)
prices.append(price.text)
ratings.append(rating.text)
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.support.select import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
# Driver
binary = FirefoxBinary()
browser = webdriver.Firefox(firefox_binary=binary)
# Variables
wait = WebDriverWait(browser, 10)
down_arrow = u'\ue015'
base_url = 'http://bpt-int.camsys-apps.com/'
# Login
def user_login(username='******', password='******'):
elem = browser.find_element_by_id('user_email')
elem.clear()
elem.send_keys(username)
elem = browser.find_element_by_id('user_password')
elem.clear()
elem.send_keys(password)
elem = browser.find_element_by_name('commit')
elem.click()
time.sleep(2)
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
caps = webdriver.DesiredCapabilities().FIREFOX
caps['marionette'] = True
binary = FirefoxBinary(r'/Applications/Firefox.app/Contents/MacOS/firefox')
# 这是下载在mac中的火狐可执行文件的默认地址
driver = webdriver.Firefox(firefox_binary=binary, capabilities=caps,
executable_path='/Users/kongweichang/Downloads/geckodriver')
url = 'https://www.baidu.com/'
# 可以改成你想显示的网页
driver.get(url)
import os
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
gecko = os.path.normpath(os.path.join(os.path.dirname(__file__),
'geckodriver'))
binary = FirefoxBinary(r'C:\Program Files\Mozilla Firefox\firefox.exe')
browser = webdriver.Firefox()
browser.maximize_window()
browser.implicitly_wait(60)
# def before_all(context):
# context.browser = webdriver.Firefox()
# context.browser.implicitly_wait(60)
# context.browser.maximize_window()
# def after_all(context):
# context.browser.quit()
# 爬杭州airbnb
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
import time
caps = webdriver.DesiredCapabilities().FIREFOX
caps["marionette"] = True
binary = FirefoxBinary(
r'D:\Program Files\Mozilla Firefox\firefox.exe') # 火狐浏览器地址
# 属性
fp = webdriver.FirefoxProfile()
fp.set_preference("permissions.default.stylesheet", 2) # 限制css
fp.set_preference("permissions.default.image", 2) # 限制图片
fp.set_preference("javascript.enabled", False) # 禁止js
#用 selenium 的 driver 来启动 firefox
driver = webdriver.Firefox(firefox_binary=binary, capabilities=caps)
for i in range(0, 5):
# 在虚拟浏览器中打开 Airbnb 页面
link = "https://zh.airbnb.com/s/homes?refinement_paths%5B%5D=%2Fhomes&query=%E4%B8%AD%E5%9B%BD%E6%B5%99%E6%B1%9F%E7%9C%81%E6%9D%AD%E5%B7%9E%E5%B8%82%E8%A5%BF%E6%B9%96%E5%8C%BA&allow_override%5B%5D=&s_tag=i-5oFEbfamp%3Bitems_offset%3D18§ion_offset=6&items_offset=" + str(
i * 18)
print(link)
driver.get(link)
#找到页面中所有的出租房
rent_list = driver.find_elements_by_css_selector('div._14csrlku')
#对于每一个出租房
def scrap_general(date_debut, date_fin):
date_1 = date_debut
date_2 = get_next_day(date_1)
print('Between ' + date_1 + ' and ' + date_2)
# creating selenium driver
binary = FirefoxBinary(r'C:\Program Files\Mozilla Firefox\firefox.exe')
executable_path = r'C:\Users\theop\Documents\Cours\Projet Option\thevoiceuk2019\data_scrapping\geckodriver.exe'
driver = webdriver.Firefox(firefox_binary=binary,
executable_path=executable_path)
with open('tweets_' + date_debut + '_' + date_fin + '.csv',
'w',
newline='',
encoding='utf-8') as csvfile:
fieldnames = [
'tweet_id', 'user_id', 'user_pseudo', 'user_name', 'date_and_time',
'content', 'nb_replies', 'nb_retweets', 'nb_jaime'
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
while date_1 != date_fin:
# reaching the result page
print('Reaching the page...')
link = 'https://twitter.com/search?l=&q=%23thevoiceuk%20since%3A' + date_1 + '%20until%3A' + date_2 + '&src=typd'
driver.get(link)
time.sleep(5)
# scroll down till the end
print('Scrolling down...')
iter = 0
id_last_tweet = driver.find_elements_by_xpath(
"//li[@data-item-type='tweet']/div")[-1].get_attribute(
'data-tweet-id')
driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
time.sleep(3)
id_last_tweet_2 = driver.find_elements_by_xpath(
"//li[@data-item-type='tweet']/div")[-1].get_attribute(
'data-tweet-id')
while id_last_tweet != id_last_tweet_2 and iter < 50:
iter += 1
id_last_tweet = id_last_tweet_2
driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
id_last_tweet_2 = driver.find_elements_by_xpath(
"//li[@data-item-type='tweet']/div")[-1].get_attribute(
'data-tweet-id')
print('We scrolled down ' + str(iter) + ' times !')
# localize each tweet
print('Finding information...')
tweets = driver.find_elements_by_xpath(
"//li[@data-item-type='tweet']/div")
print('Number of tweets : ' + str(len(tweets)))
for tweet in tweets:
tweet_info = {}
tweet_info['tweet_id'] = tweet.get_attribute('data-tweet-id')
tweet_info['user_pseudo'] = tweet.get_attribute(
'data-screen-name')
tweet_info['user_id'] = tweet.get_attribute('data-user-id')
tweet_info['user_name'] = tweet.get_attribute('data-name')
tweet_info['date_and_time'] = str(
tweet.find_element_by_xpath(
"//div[@data-tweet-id='" + tweet_info['tweet_id'] +
"']/div/div/small/a[@class='tweet-timestamp js-permalink js-nav js-tooltip']"
).get_attribute('title'))
tweet_info['content'] = tweet.find_element_by_xpath(
"//div[@data-tweet-id='" + tweet_info['tweet_id'] +
"']/div/div[@class='js-tweet-text-container']"
).text.replace('\n', ' ')
tweet_info['nb_replies'] = tweet.find_element_by_xpath(
"//span[@id='profile-tweet-action-reply-count-aria-" +
tweet_info['tweet_id'] +
"']").get_attribute('innerHTML')[0]
tweet_info['nb_retweets'] = tweet.find_element_by_xpath(
"//span[@id='profile-tweet-action-retweet-count-aria-" +
tweet_info['tweet_id'] +
"']").get_attribute('innerHTML')[0]
tweet_info['nb_jaime'] = tweet.find_element_by_xpath(
"//span[@id='profile-tweet-action-favorite-count-aria-" +
tweet_info['tweet_id'] +
"']").get_attribute('innerHTML')[0]
writer.writerow(tweet_info)
date_1 = date_2
date_2 = get_next_day(date_1)
print('\n')
print('Between ' + date_1 + ' and ' + date_2)
def setUp(self):
binary = FirefoxBinary('/usr/lib/firefox/firefox')
self.browser = webdriver.Firefox(firefox_binary=binary)
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
#Load FireFox Profile
profile = FirefoxProfile(
r'C:\Users\lapt\AppData\Roaming\Mozilla\Firefox\Profiles\g1mwnsjp.default-release'
)
url = input("Enter the profile url :") #Get Url
binary = FirefoxBinary(
"C:\\Program Files\\Mozilla Firefox\\firefox.exe") #Set Binary
# driver = webdriver.Firefox(firefox_binary=binary, executable_path="C:\\geckodriver.exe", firefox_profile=profile)
driver = webdriver.Firefox(
firefox_binary=binary,
executable_path=
"C:\\Users\\lapt\\PycharmProjects\\facebook_help_needed\\geckodriver.exe")
# driver = webdriver.Firefox()
driver.get('https://www.facebook.com')
time.sleep(10)
driver.find_element_by_xpath("//input[@name='email']").send_keys('')
password = driver.find_element_by_xpath("//input[@name='pass']")
password.send_keys('')
password.submit()
time.sleep(20)
na = url.split('https://www.facebook.com/')
print(na[1]) #Profile Name
def start_webdriver_client(self):
capabilities = {}
for c in self.capabilities:
name, value = c.split(':')
# handle integer capabilities
if value.isdigit():
value = int(value)
# handle boolean capabilities
elif value.lower() in ['true', 'false']:
value = value.lower() == 'true'
capabilities.update({name: value})
if self.proxy_host and self.proxy_port:
proxy = Proxy()
proxy.http_proxy = '%s:%s' % (self.proxy_host, self.proxy_port)
proxy.ssl_proxy = proxy.http_proxy
proxy.add_to_capabilities(capabilities)
profile = None
if self.driver.upper() == 'REMOTE':
capabilities.update(getattr(webdriver.DesiredCapabilities, self.browser_name.upper()))
if json.loads(self.chrome_options) or self.extension_paths:
capabilities = self.create_chrome_options(
self.chrome_options,
self.extension_paths).to_capabilities()
if self.browser_name.upper() == 'FIREFOX':
profile = self.create_firefox_profile(
self.firefox_preferences,
self.profile_path,
self.extension_paths)
if self.browser_version:
capabilities['version'] = self.browser_version
capabilities['platform'] = self.platform.upper()
executor = 'http://%s:%s/wd/hub' % (self.host, self.port)
try:
self.selenium = webdriver.Remote(command_executor=executor,
desired_capabilities=capabilities or None,
browser_profile=profile)
except AttributeError:
valid_browsers = [attr for attr in dir(webdriver.DesiredCapabilities) if not attr.startswith('__')]
raise AttributeError("Invalid browser name: '%s'. Valid options are: %s" % (self.browser_name, ', '.join(valid_browsers)))
elif self.driver.upper() == 'CHROME':
options = None
if self.chrome_options or self.extension_paths:
options = self.create_chrome_options(
self.chrome_options,
self.extension_paths)
if self.chrome_path:
self.selenium = webdriver.Chrome(executable_path=self.chrome_path,
chrome_options=options,
desired_capabilities=capabilities or None)
else:
self.selenium = webdriver.Chrome(chrome_options=options,
desired_capabilities=capabilities or None)
elif self.driver.upper() == 'FIREFOX':
binary = self.firefox_path and FirefoxBinary(self.firefox_path) or None
profile = self.create_firefox_profile(
self.firefox_preferences,
self.profile_path,
self.extension_paths)
self.selenium = webdriver.Firefox(
firefox_binary=binary,
firefox_profile=profile,
capabilities=capabilities or None)
elif self.driver.upper() == 'IE':
self.selenium = webdriver.Ie()
elif self.driver.upper() == 'PHANTOMJS':
self.selenium = webdriver.PhantomJS(service_args=['--ignore-ssl-errors=true'])
elif self.driver.upper() == 'OPERA':
capabilities.update(webdriver.DesiredCapabilities.OPERA)
self.selenium = webdriver.Opera(executable_path=self.opera_path,
desired_capabilities=capabilities)
else:
self.selenium = getattr(webdriver, self.driver)()
if self.event_listener is not None and not isinstance(self.selenium, EventFiringWebDriver):
self.selenium = EventFiringWebDriver(self.selenium, self.event_listener())
def get_firefox_driver_url(self, url_path, binary_path):
binary = FirefoxBinary(binary_path)
driver = webdriver.Firefox(firefox_binary=binary)
driver.get(url_path)
return driver
def main(user, password):
urlRoot = "https://es.zulutrade.com"
urlLogin = "******"
urlToScrap = "https://es.zulutrade.com/traders"
columnsFile = "ubicationColumns.json"
firefoxDirectory = r'D:\Navegadores\Mozilla Firefox\firefox.exe'
today = datetime.datetime.strftime(datetime.datetime.now(), '%Y_%m_%d')
createTodayDirectory(today)
outputFile = "zulutrade_" + today + ".csv"
columnsJson = getColumns(columnsFile)
writeHeaderFile(outputFile, columnsJson["Columns"])
options = Options()
options.add_argument("--headless")
profile = webdriver.FirefoxProfile()
profile.set_preference("dom.disable_beforeunload", True)
profile.set_preference("browser.tabs.remote.autostart", False)
profile.set_preference("browser.tabs.remote.autostart.1", False)
profile.set_preference("browser.tabs.remote.autostart.2", False)
profile.set_preference("browser.tabs.remote.force-enable", False)
profile.set_preference('browser.download.folderList', 2) # custom location
profile.set_preference('browser.download.manager.showWhenStarting', False)
profile.set_preference('browser.download.dir', os.getcwd() + '\\' + today)
profile.set_preference(
'browser.helperApps.neverAsk.saveToDisk',
"application/xml,text/xml,application/csv,application/excel,application/vnd.msexcel,application/vnd.ms-excel,text/anytext,text/comma-separated-values,text/csv,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/octet-stream"
)
profile.set_preference(
"browser.helperApps.neverAsk.openFile",
"application/xml,text/xml,application/csv,application/excel,application/vnd.msexcel,application/vnd.ms-excel,text/anytext,text/comma-separated-values,text/csv,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/octet-stream"
)
profile.set_preference("browser.helperApps.alwaysAsk.force", False)
profile.set_preference("browser.download.manager.useWindow", False)
profile.set_preference("browser.download.manager.focusWhenStarting", False)
profile.set_preference("browser.download.manager.alertOnEXEOpen", False)
profile.set_preference("browser.download.manager.showAlertOnComplete",
False)
profile.set_preference("browser.download.manager.closeWhenDone", True)
binary = FirefoxBinary(firefoxDirectory)
driver = webdriver.Firefox(firefox_options=options,
firefox_profile=profile,
firefox_binary=binary)
#driver = webdriver.Firefox(firefox_profile = profile,firefox_binary=binary)
#driver = webdriver.Firefox(firefox_profile = profile)
driver.get(urlLogin)
userElement = driver.find_element_by_id("main_tbUsername")
passwordElement = driver.find_element_by_id("main_tbPassword")
userElement.send_keys(user)
passwordElement.send_keys(password)
driver.find_element_by_id("main_btnLogin").click()
delayLogin = 30 #seconds
delay = 90 #seconds
try:
element = WebDriverWait(driver, delayLogin).until(
EC.presence_of_element_located((By.ID, 'user-top-container')))
except TimeoutException:
print("Se excedió el tiempo de espera")
driver.quit()
raise LoginException()
driver.get(urlToScrap)
try:
element = WebDriverWait(driver, delay).until(
EC.presence_of_element_located(
(By.XPATH, '//zl-load-more/button')))
except TimeoutException:
print("Se excedió el tiempo de espera")
driver.quit()
raise Exception()
moreDetailElement = driver.find_elements_by_xpath(
"//zl-performance/div/div/div/div/button")
print(len(moreDetailElement))
moreDetailElement[0].click()
for i in range(59):
print("Page: " + str(i))
try:
element = WebDriverWait(driver, delayLogin).until(
EC.presence_of_element_located(
(By.XPATH, '//zl-load-more/button')))
except TimeoutException:
print("Se excedió el tiempo de espera del boton de Cargar mas")
break
if len(driver.find_elements_by_xpath("//zl-load-more/button")) > 0:
downloadMoreElement = driver.find_element_by_xpath(
"//zl-load-more/button")
downloadMoreElement.click()
else:
break
#sleep(4.5)
rowsElements = driver.find_elements_by_xpath(
"//zl-performance-forex-list/div/table/tbody")
print(len(rowsElements))
#badgesElements = driver.find_elements_by_xpath("//zl-trader-badge")
#print(len(badgesElements))
for iRowElement in range(len(rowsElements)):
print(iRowElement)
rowData = getDataPerTrader(rowsElements[iRowElement],
columnsJson["UbicationsGrid"])
'''
numElements = len(badgesElements[iRowElement].find_elements_by_xpath(".//ngl-icon[@ng-reflect-set-icon='icon-badge-partially-verified' or @ng-reflect-set-icon='icon-badge-fully-verified']"))
print(numElements)
if numElements > 0:
print("Si hay elemento Check")
checkIconElement = badgesElements[iRowElement].find_elements_by_xpath(".//ngl-icon[@ng-reflect-set-icon='icon-badge-partially-verified' or @ng-reflect-set-icon='icon-badge-fully-verified']")[numElements - 1]
driver.execute_script("arguments[0].scrollIntoView();", rowsElements[iRowElement])
hover = ActionChains(driver).move_to_element(checkIconElement)
hover.perform()
sleep(2)
soup = BeautifulSoup(driver.page_source, 'lxml')
popUpElement = soup.find("zl-trader-verification-popover")
#print(popUpElement)
#To get lost of Focus of the little windows to iterate the next row
hover = ActionChains(driver).move_to_element(badgesElements[iRowElement])
hover.perform()
sleep(1)
'''
badgesElementsHTML = rowsElements[iRowElement].find_element_by_xpath(
".//zl-trader-badge").get_attribute('innerHTML')
for badge, item in columnsJson["UbicationsBadges"].items():
rowData[badge] = item["ICON"] in badgesElementsHTML
#open tab
driver.find_element_by_tag_name('body').send_keys(Keys.CONTROL + 't')
print(driver.window_handles)
driver.switch_to.window(driver.window_handles[1])
driver.get(rowData["Url"])
try:
element = WebDriverWait(driver, delay).until(
EC.presence_of_element_located(
(By.XPATH, '//zl-timeframes/ngl-picklist/div/button')))
except TimeoutException:
print("Se excedió el tiempo de espera")
driver.quit()
raise Exception()
rowData = getDataInsidePagePerTrader(rowData, driver,
columnsJson["UbicationsInside"])
graphicTimeElement = driver.find_element_by_xpath(
"//zl-timeframes/ngl-picklist/div/button")
graphicTimeElement.click()
graphicTotalTimeElements = driver.find_elements_by_xpath(
"//zl-timeframes/ngl-picklist/div/div/ul/li")
graphicTotalTimeElements[len(graphicTotalTimeElements) - 1].click()
excelFilename = "No hay archivo Excel disponible"
if len(
driver.find_elements_by_xpath(
"//zl-trading-history-excel-export/span/button")) > 0:
exportExcelElement = driver.find_element_by_xpath(
"//zl-trading-history-excel-export/span/button")
exportExcelElement.click()
exportExcel2007Elements = driver.find_elements_by_xpath(
"//zl-trading-history-excel-export/span/div/ul/li")
exportExcel2007Elements[0].click()
sleep(3)
excelFilename = getLastFilename(os.getcwd() + '\\' + today)
rowData["Excel"] = excelFilename
print(rowData)
dfTraders = pd.DataFrame(rowData,
columns=columnsJson["Columns"],
index=[0])
with open(outputFile, "a") as f:
dfTraders.to_csv(f,
header=None,
index=False,
encoding='ISO-8859-1',
sep='|')
# close the tab
driver.find_element_by_tag_name('body').send_keys(Keys.CONTROL + 'w')
driver.switch_to.window(driver.window_handles[0])
driver.quit()
def __init__(self):
os.environ["PATH"] += os.pathsep + GECKODRIVER_BIN
self.driver = webdriver.Firefox(
firefox_binary=FirefoxBinary(FIREFOX_BIN_PATH))
print "执行测试用例用时：", endtime - startTime
def te():
driver = webdriver.Remote(
"https://localhost:4444/wd/hub",
desired_capabilities=webdriver.DesiredCapabilities.HTMLUNIT)
driver.get('http://www.baidu.com')
el = driver.find_element_by_id('su')
print el.get_attribute('value')
driver.quit()
def seq_script(tevent):
print "a", tevent
if __name__ == '__main__':
# u = [1,2,3,4,5,6,"1","1",8,"Parent"]
# t=["132","1","2","Student"]
# runtestcase(u,t)
# te()
binary = FirefoxBinary("D:\\software\\firefox\\firefox.exe")
# 这里要指定火狐的位置，因为它不是默认位置，默认的是在C:\\Program Files（x86）\\Mozilla Firefox\\firefox.exe
driver = webdriver.Firefox(firefox_binary=binary)
# driver.implicitly_wait(30)
driver.get("http://localhost/schoolmate2/")
# 执行封装的函数
case.dologin(driver, u"test", u"test") # T3
case.click_user(driver) # T4
case.delete_user(driver)
def browser_create(self):
""" Create a browser """
self_num = random.randint(0, 99999)
myProxy = Registry().get('proxies').get_proxy()
if myProxy:
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': ''
})
self.proxy_using = True
else:
#print "No proxy"
proxy = None
self.proxy_using = False
profile_path = '/tmp/wr-selenium-{0}/'.format(self_num)
if os.path.exists(profile_path):
shutil.rmtree(profile_path)
if not os.path.exists(profile_path):
os.mkdir(profile_path)
profile = webdriver.FirefoxProfile(profile_path)
if Registry().get('config')['selenium']['css_load'] != '1':
profile.set_preference('permissions.default.stylesheet', 2)
if Registry().get('config')['selenium']['images_load'] != '1':
profile.set_preference('permissions.default.image', 2)
if Registry().get('config')['selenium']['flash_load'] != '1':
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
profile.set_preference("browser.startup.homepage", "about:blank")
profile.set_preference("startup.homepage_welcome_url", "about:blank")
profile.set_preference("startup.homepage_welcome_url.additional",
"about:blank")
fo = open('/tmp/firefox-run-{0}.log'.format(self_num), "w")
binary = FirefoxBinary(
firefox_path=Registry().get('config')['selenium']['firefox_path'],
log_file=fo)
try:
self.browser = SeleniumBrowser(
profile,
firefox_binary=binary,
ddos_phrase=self.ddos_phrase,
proxy=proxy,
ddos_human=self.ddos_human,
)
except WebDriverException:
shutil.rmtree(profile_path)
time.sleep(5)
return self.browser_create()
self.browser.set_page_load_timeout(
Registry().get('config')['selenium']['timeout_page_load'])
self.browser.implicitly_wait(
Registry().get('config')['selenium']['timeout_page_load'])
#profile.set_preference("extensions.netmonitor.har.autoConnect", True)
#profile.set_preference("extensions.netmonitor.har.contentAPIToken", "test")
#profile.set_preference("devtools.netmonitor.har.defaultLogDir", HARDIR)
#profile.set_preference("devtools.netmonitor.har.pageLoadedTimeout", 500)
except Exception as err:
print("Failed to load HAR Export Trigger extension:" + str(err) +
" - Is it in the same directory?")
firefox_options = webdriver.FirefoxOptions()
firefox_options.add_argument("-devtools")
socket.setdefaulttimeout(
SOCKETTIMEOUT
) # Sadly, this seems to be the only thing that works to actually get a timeout
try:
driver = webdriver.Firefox(firefox_profile=profile,
firefox_binary=FirefoxBinary(FIREFOX_PATH),
options=firefox_options)
except FileNotFoundError:
print("Could not find firefox binary at " + FIREFOX_PATH +
" -- trying to use default")
driver = webdriver.Firefox(firefox_profile=profile,
options=firefox_options)
time.sleep(1)
successfulruns = []
for run in range(1, TIMES + 1):
timestamp = datetime.datetime.now().strftime("%Y-%m-%d+%H-%M-%S.%f")
unixtimestamp = int(time.time())
try:
actual_label = driver.find_element_by_xpath(xpath_label).text
xpath_value = "//div[@id='analysisDiv']/table/thead/tr[{index}]/td".format(
index=index)
wait_for_element_by_xpath(xpath_value)
actual_value = driver.find_element_by_xpath(xpath_value).text
if actual_label != expected_label:
print(
"Unexpected label {label} on line {index}, possible page structure change."
.format(label=actual_label, index=index))
continue
if actual_value != expected_value:
print(
"Different value {value} for {label}. Expected value {expected_value}"
.format(value=actual_value,
label=expected_label,
expected_value=expected_value))
else:
print("Correct value {value} for {label}".format(
value=actual_value, label=actual_label))
binary = FirefoxBinary(FFBINARYPATH)
driver = webdriver.Firefox(firefox_binary=binary)
driver.get('https://www.mortgageloan.com/calculator')
fill_first_screen()
click_next()
fill_second_screen()
click_finish()
verify_values()
driver.close()
