def __init__(self, profile=None, extensions=None, user_agent=None,
profile_preferences=None, fullscreen=False, wait_time=2,
timeout=90, capabilities=None, headless=False, **kwargs):
firefox_profile = FirefoxProfile(profile)
firefox_profile.set_preference('extensions.logging.enabled', False)
firefox_profile.set_preference('network.dns.disableIPv6', False)
firefox_capabilities = DesiredCapabilities().FIREFOX
firefox_capabilities["marionette"] = True
if 'accepted_languages' in kwargs:
firefox_profile.set_preference('intl.accept_languages', kwargs['accepted_languages'])
del kwargs['accepted_languages']
if capabilities:
for key, value in capabilities.items():
firefox_capabilities[key] = value
if user_agent is not None:
firefox_profile.set_preference(
'general.useragent.override', user_agent)
if profile_preferences:
for key, value in profile_preferences.items():
firefox_profile.set_preference(key, value)
if extensions:
for extension in extensions:
firefox_profile.add_extension(extension)
if headless:
os.environ.update({"MOZ_HEADLESS": '1'})
binary = FirefoxBinary()
binary.add_command_line_options('-headless')
kwargs['firefox_binary'] = binary
self.driver = Firefox(firefox_profile,
capabilities=firefox_capabilities,
timeout=timeout, **kwargs)
if fullscreen:
ActionChains(self.driver).send_keys(Keys.F11).perform()
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__(wait_time)
class ExtensionConnection(RemoteConnection):
def __init__(self, host, firefox_profile, firefox_binary=None, timeout=30):
self.profile = firefox_profile
self.binary = firefox_binary
HOST = host
timeout = int(timeout)
if self.binary is None:
self.binary = FirefoxBinary()
if HOST is None:
HOST = "127.0.0.1"
PORT = utils.free_port()
self.profile.port = PORT
self.profile.update_preferences()
self.profile.add_extension()
self.binary.launch_browser(self.profile, timeout=timeout)
_URL = "http://%s:%d/hub" % (HOST, PORT)
RemoteConnection.__init__(
self, _URL, keep_alive=True)
def quit(self, sessionId=None):
self.execute(Command.QUIT, {'sessionId': sessionId})
while self.is_connectable():
LOGGER.info("waiting to quit")
time.sleep(1)
def connect(self):
"""Connects to the extension and retrieves the session id."""
return self.execute(Command.NEW_SESSION,
{'desiredCapabilities': DesiredCapabilities.FIREFOX})
@classmethod
def connect_and_quit(self):
"""Connects to an running browser and quit immediately."""
self._request('%s/extensions/firefox/quit' % _URL)
@classmethod
def is_connectable(self):
"""Trys to connect to the extension but do not retrieve context."""
utils.is_connectable(self.profile.port)
def __init__(self, host, firefox_profile, firefox_binary=None, timeout=30):
self.profile = firefox_profile
self.binary = firefox_binary
HOST = host
if self.binary is None:
self.binary = FirefoxBinary()
if HOST is None:
HOST = "127.0.0.1"
PORT = utils.free_port()
self.profile.port = PORT
self.profile.update_preferences()
self.profile.add_extension()
self.binary.launch_browser(self.profile)
_URL = "http://%s:%d/hub" % (HOST, PORT)
RemoteConnection.__init__(
self, _URL, keep_alive=True)
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
import datetime
if __name__ == '__main__':
caps = webdriver.DesiredCapabilities().FIREFOX
caps["marionette"] = True
binary = FirefoxBinary(r'H:\Mozilla Firefox\firefox.exe')
fp = webdriver.FirefoxProfile()
fp.set_preference("permissions.default.image",2)
driver = webdriver.Firefox(firefox_binary=binary,capabilities=caps,firefox_profile=fp)
driver.get("https://movie.douban.com/top250")
f = open('douban.txt','w',encoding='utf-8')
f.write(str(datetime.datetime.now())+'\n')
movies = []
for page in range(1,11):
commment = driver.find_elements_by_xpath("//div[@class='hd']")
for each in commment:
content = each.find_element_by_tag_name('span')
#print(content.text.strip())
movies.append(content.text.strip())
try:
next_page = driver.find_element_by_xpath("//a[@href='?start=" +str(page*25)+"&filter=']")
next_page.click()
except:
break
def test_set_binary_with_firefox_binary(options):
binary = FirefoxBinary('foo')
options.binary = binary
assert options._binary == binary
def browser_create(self, retry_counter=0):
""" Create a browser """
if retry_counter >= int(Registry().get('config')['selenium']
['browser_recreate_errors_limit']):
raise Exception(
"WebDriver can`t create browser. Check errors log selenium settings."
)
self_num = random.randint(0, 99999)
myProxy = Registry().get('proxies').get_proxy()
if myProxy:
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': ''
})
self.proxy_using = True
else:
#print "No proxy"
proxy = None
self.proxy_using = False
profile_path = '/tmp/wr-selenium-{0}/'.format(self_num)
if os.path.exists(profile_path):
shutil.rmtree(profile_path)
if not os.path.exists(profile_path):
os.mkdir(profile_path)
profile = webdriver.FirefoxProfile(profile_path)
if Registry().get('config')['selenium']['css_load'] != '1':
profile.set_preference('permissions.default.stylesheet', 2)
if Registry().get('config')['selenium']['images_load'] != '1':
profile.set_preference('permissions.default.image', 2)
if Registry().get('config')['selenium']['flash_load'] != '1':
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
profile.set_preference("browser.startup.homepage", "about:blank")
profile.set_preference("browser.formfill.enable", "false")
profile.set_preference("startup.homepage_welcome_url", "about:blank")
profile.set_preference("startup.homepage_welcome_url.additional",
"about:blank")
if myProxy and len(myProxy):
agent_IP, agent_Port = myProxy.split(":")
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.share_proxy_settings", True)
profile.set_preference("network.http.use-cache", False)
profile.set_preference("network.proxy.http", agent_IP)
profile.set_preference("network.proxy.http_port", int(agent_Port))
profile.set_preference('network.proxy.ssl_port', int(agent_Port))
profile.set_preference('network.proxy.ssl', agent_IP)
profile.set_preference('network.proxy.socks', agent_IP)
profile.set_preference('network.proxy.socks_port', int(agent_Port))
fo = open('/tmp/firefox-run-{0}.log'.format(self_num), "w")
binary = FirefoxBinary(
firefox_path=Registry().get('config')['selenium']['firefox_path'],
log_file=fo)
try:
self.browser = SeleniumBrowser(
profile,
firefox_binary=binary,
proxy=proxy,
browser_wait_re=self.browser_wait_re,
)
except WebDriverException as e:
self.logger.ex(e)
self.logger.log("Re-trying. Browser creation error: " + str(e))
shutil.rmtree(profile_path)
time.sleep(5)
retry_counter += 1
return self.browser_create(retry_counter)
self.browser.set_page_load_timeout(
Registry().get('config')['selenium']['timeout_page_load'])
self.browser.implicitly_wait(
Registry().get('config')['selenium']['timeout_page_load'])
def before_feature(context, feature):
binary = FirefoxBinary('firefox')
context.ff = webdriver.Firefox(firefox_binary=binary)
from selenium import webdriver
import time
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.keys import Keys
binary = FirefoxBinary('') #location of firefox.exe
b = webdriver.Firefox(firefox_binary=binary)
b.get('http://web.whatsapp.com')
raw_input()
print("Success!!!")
elem = b.find_element_by_xpath('//span[contains(text(),"Deeksha")]')
elem.click()
elem1 = b.find_elements_by_class_name('input')
while True:
elem1[1].send_keys('hi')
b.find_element_by_class_name('send-container').click()
def __init__(self):
self.server_name = 'qa.orcid.org'
self.signin_page = 'https://%s/signin' % self.server_name
self.auth_page = 'https://%s/signin/auth.json' % self.server_name
ff_bin = FirefoxBinary('/opt/firefox-56.0.2/firefox')
self.ff = webdriver.Firefox(firefox_binary=ff_bin)
output_filename = 'reviews_%s.json.gz'
restaurants = read_json('restaurants.json')
restaurants_data = read_json('restaurants_data.json', {})
reviews = {} # initialize as a void dictionary
skip_filename = 'skip_restaurants.json'
skip_restaurants = set(read_json(skip_filename, []))
# If there's at least a restaurant in skip_restaurants,
# only those restaurants will be downloaded
total = len(restaurants_data)
with Display(visible=0, size=(1024, 768)) as display:
firefox_binary = FirefoxBinary(config['firefox_binary'])
def firefox():
browser = Firefox(executable_path=config['executable_path'],
firefox_binary=firefox_binary)
browser.set_page_load_timeout(30)
return browser
try:
ff = firefox()
for n, (k, d) in enumerate(restaurants_data.items()):
if k in skip_restaurants: continue # already processed
if d['rating'] is None:
#pdf text data
filename = input('Enter filename:')
#filename='Historica' + '.pdf'
text = convert_pdf_to_txt(filename)
data = []
n = 2500
pattern = "[^\x20-\x7E]+"
for i in range(0, len(text), n):
data.append("".join(text[i:i + n]))
#connect webdriver
#driver = webdriver.Chrome(executable_path=r"E:\Data Science\eng_to_jap\chromedriver.exe")
gecko = os.path.normpath(os.path.join(os.path.dirname(__file__),
'geckodriver'))
binary = FirefoxBinary(r'C:\Program Files\Mozilla Firefox\firefox.exe')
driver = webdriver.Firefox(firefox_binary=binary,
executable_path=gecko + '.exe')
#driver = webdriver.Firefox('geckodriver')
#url='https://deepl.com'
url = 'https://www.deepl.com/translator#en/ja'
driver.get(url)
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
# //tag[@class=''] class is attribute and inside '' is value i.e. //tag[@Attribute='Value']
WebDriverWait(driver, 20).until(
EC.element_to_be_clickable(
(By.XPATH,
import selenium
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
import os, sys, time
# 1- set profile
#profile = os.path.dirname(sys.argv[0]) + "/selenita"
binary = FirefoxBinary('C:/Program Files (x86)/Mozilla Firefox/firefox.exe')
profile = 'dependencies\\1ndsjjut.default'
fp = webdriver.FirefoxProfile(profile)
driver = webdriver.Firefox(firefox_profile=fp, firefox_binary=binary)
# 2- get tmp file location
profiletmp = driver.firefox_profile.path
# but... the current profile is a copy of the original profile :/
print("running profile " + profiletmp)
driver.get("http://httpbin.org")
time.sleep(2)
input("Press a key when finish doing things") # I've installed an extension
# 3- then save back
print("saving profile " + profiletmp + " to \\" + profile)
if os.system("copy " + profiletmp + "\ " + profile):
print("files should be copied :/")
driver.quit()
sys.exit(0)
if os.path.isfile(output_file):
exit("Output File Exists !")
def screenshot_el(el, epoch):
fname = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(epoch)))
image = el.screenshot_as_png
with open("./screenshot/{}.png".format(fname), 'wb') as out:
out.write(image)
profile = webdriver.FirefoxProfile()
profile.set_preference('dom.webnotifications.enabled', False)
binary = FirefoxBinary('/usr/bin/firefox')
binary.add_command_line_options('--headless')
driver = webdriver.Firefox(profile, firefox_binary=binary)
driver.get('https://m.facebook.com/')
input_username = driver.find_elements_by_css_selector('input#m_login_email')
input_password = driver.find_elements_by_css_selector('input[type="password"]')
submit_btn = driver.find_elements_by_css_selector('input[name="login"]')
if len(input_username) > 0 and len(input_password) > 0 and len(submit_btn) > 0:
print("Login")
user_username = input("Username : "******"Password : ")
input_username[0].send_keys(user_username)
#run_local.py
#by drh4kor
#inception date:2019-2-14
#get reference to the libraries
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
#set variables
loginURL = 'http://10.10.1.20/'
binaryPath = '/usr/bin/firefox'
#create object
binary = FirefoxBinary(binaryPath)
#create browser
driver = webdriver.Firefox(firefox_binary=binary)
driver.implicitly_wait(60)
#navigate to log in page
driver.get(loginURL)
#get the textbox to login
inputElement_id = driver.find_element_by_name('login_id')
#get the textbox for password
inputElement_psw = driver.find_element_by_name('login_pwd')
#set the path to the link element does not have id or name property.
#pathToLink='/html/body/table/tbody/tr/td/table/tbody/tr[2]/td/table/tbody/tr[4]/td/table/tbody/tr[2]/td[2]/table/tbody/tr[2]/td/table/tbody/tr[2]/td[2]/table/tbody/tr[5]/td[2]/table/tbody/tr/td/a'
#inputElement_link=driver.find_element_by_xpath(pathToLink)
#inputElement_link=driver.find_element_by_xpath("/html/body/table/tbody/tr/td/table/tbody/tr[2]/td/table/tbody/tr[4]/td/table/tbody/tr[2]/td[2]/table/tbody/tr[2]/td/table/tbody/tr[2]/td[2]/table/tbody/tr[5]/td[2]/table/tbody/tr/td/a")
inputElement_link = driver.find_element_by_xpath("//*[@id='btn_login']")
#inputElement_link=driver.find_element_by_link_text('Login')
#setup counter variables
def parse(self, response):
binary = FirefoxBinary('C:/Program Files/Mozilla Firefox/firefox.exe')
browser = webdriver.Firefox(firefox_binary=binary)
browser.get(response.url)
print(browser.page_source)
try:
ap = argparse.ArgumentParser()
ap.add_argument(
"-d",
"--driver",
type=str,
default="phantomjs",
help="which driver to use [option: phantomjs, firefox, chrome]")
args = vars(ap.parse_args())
choice = args["driver"]
driver = ""
if choice == "firefox":
binary = FirefoxBinary('firefox')
driver = webdriver.Firefox(firefox_binary=binary)
elif choice == "chrome":
driver = webdriver.Chrome()
elif choice == "phantomjs":
driver = webdriver.PhantomJS(executable_path=r'phantomjs')
else:
print("Invalid Choice")
sys.exit(1)
driver.set_window_size(1120, 550)
driver.get("http://en.savefrom.net")
shURL = driver.find_element_by_xpath(
'//input[@id="sf_url" and @type="text"]')
shURL.send_keys(url)
rPort = itm.find('RedisPort').text
db = itm.find('Database').text
user1 = itm.find('userName').text
password1 = itm.find('pass').text
collName = itm.find('collection').text
errCollName = itm.find('collection2').text
# Client = MongoClient(dbIP)
Client = MongoClient(shardIP)
db = Client[db]
# db.authenticate(user1, password1)
collection1 = db[collName]
collection3 = db[errCollName]
r = redis.StrictRedis(host=rIP, port=rPort)
binary = FirefoxBinary('/usr/local/desktop/firefox')
capabilities = webdriver.DesiredCapabilities().FIREFOX
capabilities["-marionette"] = False
fox = []
fox.append('http://feeds.news24.com/articles/news24/TopStories/rss')
def error(err):
try:
source = "News 24"
time = datetime.datetime.now()
time = time.isoformat()
time = arrow.get(time).datetime
collection3.insert([{
__author__ = 'Bobby Williams'
import pyautogui as pa
import time
from getpass import getpass
import os
import sys
from datetime import datetime
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
username = '******'
binary = FirefoxBinary(r'C:\XXX\Mozilla Firefox\firefox.exe')
success_log = []
def get_creds():
#### Get login creds. ####
global pw
pw = getpass('Enter password for \'{}\': '.format(username))
def find_configs():
#### Get a list of config files ####
# Directory where the configs reside:
config_dir = r'C:\XXX\XXX-fw-cfg-gen\generated'
print('Checking for configs in: {} ...'.format(config_dir))
os.chdir(config_dir)
files = os.listdir('.')
for i in files:
def download_download_usgs(outputfile):
fp = webdriver.FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.dir", dir_out)
fp.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"application/msword, application/csv, application/ris, text/csv, image/png, application/pdf, text/html, text/plain, application/zip, application/x-zip, application/x-zip-compressed, application/download, application/octet-stream"
)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.useDownloadDir", True)
fp.set_preference("browser.helperApps.alwaysAsk.force", False)
fp.set_preference("browser.download.manager.alertOnEXEOpen", False)
fp.set_preference("browser.download.manager.closeWhenDone", True)
fp.set_preference("browser.download.manager.showAlertOnComplete", False)
fp.set_preference("browser.download.manager.useWindow", False)
fp.set_preference(
"services.sync.prefs.sync.browser.download.manager.showWhenStarting",
False)
fp.set_preference("pdfjs.disabled", True)
fp.set_preference("browser.link.open_newwindow.override.external", True)
#C:\Program Files (x86)\Mozilla Firefox
binary = FirefoxBinary("C://Program Files//Mozilla Firefox//firefox.exe")
driver = webdriver.Firefox(firefox_profile=fp,
firefox_binary=binary,
executable_path='C://geckodriver')
wait = WebDriverWait(driver, 10)
driver.minimize_window()
url = 'https://sfbay.wr.usgs.gov/access/wqdata/query/expert.html'
driver.get(url)
time.sleep(1)
# 1 - Columns to Show
for var in query:
if int(var) <= 17:
driver.find_element_by_xpath(
"/html/body/table[2]/tbody/tr/th/table/tbody/tr/th[2]/span/form/table/tbody/tr/td/table/tbody/tr[1]/td[1]/font/input[@value='"
+ VARS[var] + "']").click()
elif 17 < int(var) & int(var) <= 20:
driver.find_element_by_xpath(
"/html/body/table[2]/tbody/tr/th/table/tbody/tr/th[2]/span/form/table/tbody/tr/td/table/tbody/tr[1]/td[1]/table/tbody/tr/td[1]/font/input[@value='"
+ VARS[var] + "']").click()
else:
driver.find_element_by_xpath(
"/html/body/table[2]/tbody/tr/th/table/tbody/tr/th[2]/span/form/table/tbody/tr/td/table/tbody/tr[1]/td[1]/table/tbody/tr/td[2]/font/input[@value='"
+ VARS[var] + "']").click()
# 3 - Sort by:
name1 = 'sort1'
value_3 = 'fulldate'
name2 = 'sort2'
value_4 = 'stat'
name3 = 'sort3'
value_5 = 'depth'
driver.find_element_by_xpath(
"/html/body/table[2]/tbody/tr/th/table/tbody/tr/th[2]/span/form/table/tbody/tr/td/table/tbody/tr[4]/td[1]/select/option[2]"
).click()
driver.find_element_by_xpath(
"/html/body/table[2]/tbody/tr/th/table/tbody/tr/th[2]/span/form/table/tbody/tr/td/table/tbody/tr[5]/td[1]/select/option[3]"
).click()
driver.find_element_by_xpath(
"/html/body/table[2]/tbody/tr/th/table/tbody/tr/th[2]/span/form/table/tbody/tr/td/table/tbody/tr[6]/td[1]/select/option[5]"
).click()
#4 - Table Format
name4 = 'out'
value = 'html' #comma
name5 = 'param'
name6 = 'maxrow' # 10, 50, 100, 500, 1000, "99999"
driver.find_element_by_xpath(
"/html/body/table[2]/tbody/tr/th/table/tbody/tr/th[2]/span/form/table/tbody/tr/td/table/tbody/tr[8]/th/table/tbody/tr/td/select/option[1]"
).click()
driver.find_element_by_xpath(
"/html/body/table[2]/tbody/tr/th/table/tbody/tr/th[2]/span/form/table/tbody/tr/td/table/tbody/tr[8]/th/table/tbody/tr/td/font/select/option[5]"
).click()
#5 - Submit
type7 = 'submit'
name7 = "Query Database"
driver.find_element_by_xpath(
"/html/body/table[2]/tbody/tr/th/table/tbody/tr/th[2]/span/form/table/tbody/tr/td/table/tbody/tr[8]/th/table/tbody/tr/th/input[2]"
).click()
alert = driver.switch_to_alert()
alert.accept()
# Get Data:
info = wait.until(
ec.visibility_of_element_located(
(By.XPATH, "/html/body/form[2]/table/tbody/tr/td[1]"))).text
noData = info.split('out of ')[1][:-10]
countData = info.split('out of ')[0].split('to ')[1][:-1]
subname = '10p'
n = 1
w = 1
while countData != noData:
table = driver.find_element_by_xpath('/html/body/table[3]')
extractor = Extractor(table.get_attribute("outerHTML")).parse()
extractor.write_to_csv(path='.')
if float(countData) > float(noData) * 0.1 * n:
n += 1
subname = str(n) + '0p'
out_file = subname + '_' + outputfile
if os.path.exists(os.path.join(os.getcwd(), out_file)):
df = pd.read_csv(os.path.join(os.getcwd(), out_file))
data = pd.read_csv(os.path.join(os.getcwd(), 'output.csv'),
encoding='cp1252')
df = pd.concat([df, data], ignore_index=True)
df.to_csv(os.path.join(os.getcwd(), out_file))
else:
data = pd.read_csv(os.path.join(os.getcwd(), 'output.csv'),
encoding='cp1252')
data.to_csv(os.path.join(os.getcwd(), out_file))
info = wait.until(
ec.visibility_of_element_located(
(By.XPATH, "/html/body/form[2]/table/tbody/tr/td[1]"))).text
countData = info.split('out of ')[0].split('to ')[1][:-1]
print(countData)
print(w)
w += 1
try:
driver.find_element_by_xpath(
"/html/body/form[2]/table/tbody/tr/td[1]/input").click()
except:
print(
"Programa encerrado. Checar se todos os dados foram baixados")
return Data
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
import sys
import time
import urllib.request
import os
# import pyfiglet
from ctypes import *
import multiprocessing
import asyncio
import threading
path = "H:\Parsing\chromedriver.exe"
# options = webdriver.ChromeOptions()
# options.add_argument("--disable-setuid-sandbox")
# options.add_argument('headless')
binary = FirefoxBinary(
r'C:\Program Files\Firefox Developer Edition\firefox.exe')
driver = webdriver.Firefox(executable_path=r"C:\Twitter_Likes\geckodriver.exe",
log_path=r"C:\Twitter_Likes\geckodriver.log")
driver.get("https://twitter.com/i/likes")
os.system('cls')
countTemp = 0
ScanCount = 0
DownSuccess = 0
SuccessCount = 0
DownCount = 0
ImageCount = 0
imageLinks = []
RmvimageLinks = []
TempLinks = []
Temp = False
import DBkeywords
import test_DB
import currentdate
import getbing
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# sys.setdefaultencoding() does not exist, here!
# path to the firefox binary inside the Tor package
binary = '/Applications/TorBrowser.app/Contents/MacOS/firefox'
if os.path.exists(binary) is False:
raise ValueError("The binary path to Tor firefox does not exist.")
firefox_binary = FirefoxBinary(binary)
firefoxProfile = FirefoxProfile()
firefoxProfile.set_preference('thatoneguydotnet.QuickJava.curVersion', '2.1.0') ## Prevents loading the 'thank you for installing screen'
firefoxProfile.set_preference('thatoneguydotnet.QuickJava.startupStatus.Images', 2) ## Turns images off
firefoxProfile.set_preference('thatoneguydotnet.QuickJava.startupStatus.AnimatedImage', 2) ## Turns animated images off
firefoxProfile.set_preference("thatoneguydotnet.QuickJava.startupStatus.CSS", 2) ## CSS
firefoxProfile.set_preference("thatoneguydotnet.QuickJava.startupStatus.Cookies", 2) ## Cookies
firefoxProfile.set_preference("thatoneguydotnet.QuickJava.startupStatus.Flash", 2) ## Flash
firefoxProfile.set_preference("thatoneguydotnet.QuickJava.startupStatus.Java", 2) ## Java
firefoxProfile.set_preference("thatoneguydotnet.QuickJava.startupStatus.JavaScript", 2) ## JavaScript
firefoxProfile.set_preference("thatoneguydotnet.QuickJava.startupStatus.Silverlight", 2)
browser = None
def get_browser(binary=None,profile=None):
global browser
def findBookUrl(self):
directory_name = '.'
binary = FirefoxBinary('/docs/python_projects/firefox/firefox')
fp = webdriver.FirefoxProfile()
fp.set_preference("webdriver.log.file", "/tmp/firefox_console");
fp.set_preference("browser.download.folderList", 2)
fp.set_preference('browser.download.manager.showWhenStarting', False)
fp.set_preference('browser.download.manager.focusWhenStarting', False)
fp.set_preference("browser.download.dir", directory_name)
fp.set_preference("browser.download.manager.scanWhenDone", False)
fp.set_preference("browser.download.manager.useWindow", False)
# fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/octet-stream")
fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/octet-stream,application/xml,application/pdf,text/plain,text/xml,image/jpeg,text/csv,application/zip,application/x-rar-compressed");
fp.set_preference("browser.helperApps.alwaysAsk.force", False);
fp.set_preference("browser.popups.showPopupBlocker", False);
fp.update_preferences()
driver = webdriver.Firefox(firefox_profile=fp, firefox_binary=binary)
# driver.find_element_by_xpath("html/body/table/tbody/tr[2]/td/div/table/tbody/tr/td[1]/img")
driver.get(self.baseUrl)
efd_link = driver.find_element_by_css_selector(".login-popup > div:nth-child(1)")
efd_link.click()
try:
emailEl = driver.find_element_by_css_selector('#packt-user-login-form > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > input:nth-child(1)')
# emailEl = driver.find_element_by_name("email")
'''
Login with user credential
'''
emailEl.send_keys('*****@*****.**')
passwordEl = driver.find_element_by_css_selector("#packt-user-login-form > div:nth-child(1) > div:nth-child(1) > div:nth-child(2) > div:nth-child(1) > input:nth-child(1)")
passwordEl.send_keys('default')
loginEl = driver.find_element_by_css_selector("#packt-user-login-form > div:nth-child(1) > div:nth-child(1) > div:nth-child(3) > input:nth-child(1)")
loginEl.click()
if True:
'''
clicking on My Account
'''
myAccountEl = driver.find_element_by_css_selector('#account-bar-logged-in > a:nth-child(1) > div:nth-child(1) > strong:nth-child(1)')
myAccountEl.click()
'''
clicking My ebooks
'''
myEbook = driver.get(self.baseUrl + 'account/my-ebooks')
productListEls = driver.find_elements_by_css_selector('div.product-line')
print len(productListEls)
bookList = list()
for productEl in productListEls:
print productEl
try:
bookName = productEl.find_element_by_css_selector('.title').text
book = self.createBookDetail(bookName)
productEl.click()
readMeEl = productEl.find_element_by_css_selector('.fake-button-text')
print 'new page',
isbnEl = productEl.find_elements_by_css_selector('div > div:nth-child(2) > div:nth-child(1)> a:nth-child(1) > div:nth-child(1)')
book.isbn_13 = isbnEl[0].get_attribute('isbn')
# readMeEl.click()
print 'div.product-line:nth-child(1) > div:nth-child(2) > div:nth-child(1) > a:nth-child(1) > div:nth-child(1)',
# readMeEl.find_element_by_css_selector('h2.ng-binding')
#
# readingEl = driver.get('https://www.packtpub.com/mapt/book/All%20Books/' + book.isbn_13)
# bookName1=driver.find_elements_by_css_selector('h2.ng-binding')[0].text
bookList.append(book)
except Exception as e:
print e
# product_account_list_el=driver.find_elements_by_css_selector('#product-account-list')
driver.get('https://www.packtpub.com/packt/offers/free-learning')
try:
'''
clicking on Claim your free ebook
'''
bookNameEl_1 = driver.find_element_by_css_selector('.dotd-title > h2:nth-child(1)')
isBookAlreadyAvailable = False
bookName_1 = bookNameEl_1.text
for book in bookList:
if bookName_1 in book.bookName:
isBookAlreadyAvailable = True
break
if not isBookAlreadyAvailable:
claimFreeEbookEl = driver.find_element_by_css_selector('.book-claim-token-inner > input:nth-child(3)')
claimFreeEbookEl.click()
except Exception as e:
print e
# myEbook.click()
except Exception as e:
print e
finally:
print 'completed'
print 'hi'
def launch_browser(url):
'''
Launch a new browser, and set the parameters for the browser.
'''
if env.threadlocal.TESTING_BROWSER.upper() == 'FIREFOX':
firefox_capabilities = DesiredCapabilities.FIREFOX
## set profile
fp = webdriver.FirefoxProfile()
fp.set_preference('browser.download.manager.showWhenStarting', False)
try:
env.THREAD_LOCK.acquire()
if env.FIREFOX_BINARY == '':
browser = webdriver.Firefox(
executable_path=env.DRIVER_OF_FIREFOX,
firefox_profile=fp,
capabilities=firefox_capabilities)
else:
browser = webdriver.Firefox(
executable_path=env.DRIVER_OF_FIREFOX,
firefox_profile=fp,
firefox_binary=FirefoxBinary(
firefox_path=env.FIREFOX_BINARY))
if env.threadlocal.TESTING_BROWSER not in env.BROWSER_VERSION_INFO:
env.BROWSER_VERSION_INFO[
env.threadlocal.
TESTING_BROWSER] = browser.capabilities['browserVersion']
except:
log.handle_error()
return False
finally:
env.THREAD_LOCK.release()
elif env.threadlocal.TESTING_BROWSER.upper() == 'CHROME':
try:
env.THREAD_LOCK.acquire()
browser = webdriver.Chrome(executable_path=env.DRIVER_OF_CHROME)
if env.threadlocal.TESTING_BROWSER not in env.BROWSER_VERSION_INFO:
env.BROWSER_VERSION_INFO[
env.threadlocal.
TESTING_BROWSER] = browser.capabilities['version']
except:
log.handle_error()
return False
finally:
env.THREAD_LOCK.release()
elif env.threadlocal.TESTING_BROWSER.upper() == 'IE':
'''
os.popen('TASKKILL /F /IM IEDriverServer.exe')
dc = DesiredCapabilities.INTERNETEXPLORER.copy()
dc['nativeEvents'] = False
dc['acceptSslCerts'] = True
'''
try:
env.THREAD_LOCK.acquire()
browser = webdriver.Ie(executable_path=env.DRIVER_OF_IE)
if env.threadlocal.TESTING_BROWSER not in env.BROWSER_VERSION_INFO:
env.BROWSER_VERSION_INFO[
env.threadlocal.
TESTING_BROWSER] = browser.capabilities['version']
except:
log.handle_error()
return False
finally:
env.THREAD_LOCK.release()
browser.set_window_size(1366, 758)
browser.set_window_position(0, 0)
browser.set_page_load_timeout(300)
browser.implicitly_wait(0)
browser.get(url)
return browser
def setup_driver():
binary = FirefoxBinary(
'C:\\Program Files (x86)\\Mozilla Firefox\\firefox.exe')
driver = webdriver.Firefox(firefox_binary=binary)
return driver
def getWebDriver(cls, **kwargs):
firefox_binary = FirefoxBinary(firefox_path=cls.firefox_path)
return super(NoHTML5SeleniumTestCase,
cls).getWebDriver(firefox_binary=firefox_binary)
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.keys import Keys
user = sys.argv[1]
password = sys.argv[2]
gni_user = sys.argv[3]
gni_password = sys.argv[4]
head = sys.argv[5]
acount_number = sys.argv[6]
do_you_wanna_wait = sys.argv[7]
option = Options()
option.headless = True
firefox_binary = FirefoxBinary('/usr/lib/firefox/firefox')
#config functions
def init_bot():
print('initializing')
if (head == '--headless'):
firefox = webdriver.Firefox(options=option,
firefox_binary=firefox_binary)
return firefox
elif (head == '--head'):
firefox = webdriver.Firefox(firefox_binary=firefox_binary)
return firefox
print('initialized')
from selenium.webdriver import Firefox
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
print("Masukkan Npm Anda:")
npm = input()
print("Masukkan Password SIAP Anda:")
paswd = input('')
opsi = Options()
opsi.headless = False
binary = FirefoxBinary("C:\\Program Files\\Mozilla Firefox\\firefox.exe")
cap = DesiredCapabilities().FIREFOX
cap['marionette'] = True
browser = Firefox(executable_path='geckodriver.exe',
options=opsi,
capabilities=cap,
firefox_binary=binary)
browser.get('http://siap.poltekpos.ac.id/siap/besan.depan.php')
name = browser.find_element_by_name('user_name')
word = browser.find_element_by_name('user_pass')
login = browser.find_element_by_name('login')
name.send_keys(npm)
word.send_keys(paswd)
login.click()
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
binary = FirefoxBinary('/usr/bin/geckodriver')
browser = webdriver.Firefox(firefox_binary=binary)
# Created by Jianguo on 2017/10/27 __author__ = "Jianguo Jin ([email protected])" """ Description: 使用Firefox 浏览器 """ import os from time import sleep from selenium import webdriver from selenium.webdriver.firefox.firefox_binary import FirefoxBinary # 制定火狐浏览器的二进制.exe 文件路径 ff_bin = FirefoxBinary(r'C:\Program Files (x86)\Mozilla Firefox\firefox.exe') # 配置FF配置文件 ff_profile = webdriver.FirefoxProfile() driver = webdriver.Firefox(firefox_binary=ff_bin) driver.maximize_window() url = 'https://www.tmall.com/' driver.get(url) search_field = driver.find_element_by_name('q') search_field.clear() search_field.send_keys('小米 Note') search_field.submit()
from selenium import webdriver
from pyvirtualdisplay import Display
display = Display(visible=0, size=(800, 600))
display.start()
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
binary = FirefoxBinary("/root/Downloads/firefox/firefox-bin")
driver = webdriver.Firefox(firefox_binary=binary)
driver.get("http://www.devdungeon.com")
driver.save_screenshot("/root/test.png")
driver.close()
display.stop()
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0
from selenium.webdriver.support import expected_conditions as EC # available since 2.26.0
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
import time
import os
binary = FirefoxBinary('/usr/bin/firefox')
profile = FirefoxProfile()
profile.set_preference("browser.download.panel.shown", "false")
profile.set_preference("browser.download.manager.showWhenStarting", "false")
profile.set_preference(
"browser.helperApps.neverAsk.openFile",
"application/octet-stream,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
)
profile.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"application/octet-stream,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
)
def wait_clickable(xpath, wait=True):
try:
if wait:
element = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, xpath)))
el.find_element_by_xpath(
".//*[contains(@class,'course-card--course-meta-info')]/span[position()=3]"
).text for el in elements_collection
]
page_dataframe['Difficulty'] = levels
return page_dataframe
# %%
# - Create driver with local binaries
try:
driver_location = r"{0}/webdriver/geckodriver.exe".format(os.getcwd())
# - Disable images for fsater scraping
firefox_profile = webdriver.FirefoxProfile()
firefox_profile.set_preference('permissions.default.image', 2)
firefox_binary = FirefoxBinary(
r"{0}/FirefoxPortable/App/Firefox64/firefox.exe".format(os.getcwd()))
driver = webdriver.Firefox(firefox_binary=firefox_binary,
executable_path=driver_location,
firefox_profile=firefox_profile)
except Exception as err:
print('Sorry, but there was an error in launching the browser.')
exit(0)
driver.implicitly_wait(timeout)
#print("driver OK")
# %%
# - main try/catch (try)
try:
#display = XvncDisplay(rfbport='5903')
#display.start()
import os
os.environ['DISPLAY'] = ':1'
from pyvirtualdisplay import Display
display = Display(visible=0, size=(1024, 768))
display.start()
log_dir = os.path.join('/root/tmp/firefox')
try:
os.makedirs(log_dir)
except OSError, e:
pass
log_path = os.path.join(
log_dir, '{}.log'.format(datetime.datetime.now().isoformat('_')))
log_file = open(log_path, 'w')
binary = FirefoxBinary(firefox_path='/usr/bin/firefox', log_file=log_file)
ffprofile = webdriver.FirefoxProfile()
adblockfile = '/usr/bin//[email protected]'
ffprofile.add_extension(adblockfile)
ffprofile.set_preference("extensions.adblockplus.currentVersion", "4")
#ffprofile.set_preference("general.useragent.override", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1")
#wd = webdriver.Firefox(ffprofile)
#dcap = {}
#mydriver = None
#browser_agent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
#dcap = dict(DesiredCapabilities.FIREFOX)
#dcap["firefox.page.settings.userAgent"] = browser_agent
#mydriver = webdriver.PhantomJS(desired_capabilities=dcap)
#print(dcap)
def __init__(
self,
profile=None,
extensions=None,
user_agent=None,
profile_preferences=None,
fullscreen=False,
wait_time=2,
timeout=90,
capabilities=None,
headless=False,
incognito=False,
**kwargs
):
firefox_profile = FirefoxProfile(profile)
firefox_profile.set_preference("extensions.logging.enabled", False)
firefox_profile.set_preference("network.dns.disableIPv6", False)
firefox_capabilities = DesiredCapabilities().FIREFOX
firefox_capabilities["marionette"] = True
firefox_options = Options()
if capabilities:
for key, value in capabilities.items():
firefox_capabilities[key] = value
if user_agent is not None:
firefox_profile.set_preference("general.useragent.override", user_agent)
if profile_preferences:
for key, value in profile_preferences.items():
firefox_profile.set_preference(key, value)
if extensions:
for extension in extensions:
firefox_profile.add_extension(extension)
if headless:
os.environ.update({"MOZ_HEADLESS": "1"})
binary = FirefoxBinary()
binary.add_command_line_options("-headless")
kwargs["firefox_binary"] = binary
if incognito:
firefox_options.add_argument("-private")
self.driver = Firefox(
firefox_profile,
capabilities=firefox_capabilities,
options=firefox_options,
timeout=timeout,
**kwargs
)
if fullscreen:
ActionChains(self.driver).send_keys(Keys.F11).perform()
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__(wait_time)
