def __init__(self): self.start_urls = ['http://www.amazon.com/review/top-reviewers?page=' + str(self.start)] self.allowed_domains = ['www.amazon.com'] # self.profile = FirefoxProfile('/home/romitas/.mozilla/firefox/68h3udd9.AmazonScraper') profile = FirefoxProfile() profile.set_preference('network.protocol-handler.external.mailto', False) self.driver = webdriver.Firefox(self.profile) self.driver_login()
def __init__(self, category=None, *args, **kwargs): super(DmozSpider, self).__init__(*args, **kwargs) self.selection = [kwargs.get('selection')][0] self.logger.info("Selection is: %s", str(self.selection)) if self.selection != 'active' and self.selection != 'retired': sys.exit("Invalid selection") elif self.selection == None: sys.exit("Selection not set") self.output_dir = [kwargs.get('output_dir')][0] self.logger.info("Output directory is: %s", str(self.output_dir)) if self.output_dir == None: sys.exit("No output directory specified") self.position_type = [kwargs.get('position_type')][0] self.logger.info("Position type is: %s", str(self.position_type)) if self.position_type != 'qb' and self.position_type != 'wr' and self.position_type != 'rb' and self.position_type != 'te': sys.exit("Invalid position type") collect_csvs = [kwargs.get('collect_csvs')][0] self.logger.info("Collect csvs is set to: %s", str(collect_csvs)) if collect_csvs == None: sys.exit("Collect csvs is not set!") elif collect_csvs == "True": self.collect_csvs = True elif collect_csvs == "False": self.collect_csvs = False else: sys.exit("Collect csvs set to invalid value") if self.selection == 'retired': self.old_guys = [kwargs.get('old_guys')][0] self.logger.info("Old Guys is set to %s", str(self.old_guys)) if self.old_guys == None: sys.exit("Old guys not set!") elif self.old_guys != "True" and self.old_guys != "False": sys.exit("Old guys set to invalid value!") # If they had college stats self.gameLogsXPathCollege = "//*/*[1]/*/*[3]/a[contains(.,'Gamelogs')]" # No college stats self.gameLogsXPathNoCollege = "//*/*[1]/*/*[2]/a[contains(.,'Gamelogs')]" # Initialize a new Firefox webdriver profile = FirefoxProfile("/Users/andrasta/Desktop/InfoVis/project/FirefoxProfileUpdated") profile.set_preference('browser.download.folderList', 2) profile.set_preference('browser.download.dir', os.getcwd()+"/"+self.output_dir) #os.getcwd()+"/QB") profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'text/csv') profile.set_preference('browser.helperApps.neverAsk.openFile', 'text/csv') self.driver = webdriver.Firefox(firefox_profile=profile) # Doesn't matter # self.driver.maximize_window() self.wait = WebDriverWait(self.driver, 5) self.no_stat_players = []
def setUp(self): StaticLiveServerTestCase.setUp(self) profile = FirefoxProfile() # Browser itself attempts to validate form fields before they are sent to django. # Fields where input type="Number" accept "100.0" when locale is "en" and "100,0" when locale is "fi", and when # they reject the value, django sees an empty value instead. # To prevent this causing more problems, force browser used by tests to use same locale as django, typically # "en". # We may want to occassionally test with other locales, so localize_input, number_format etc. when entering # and reading decimals/floats. profile.set_preference("intl.accept_languages", get_language()) profile.set_preference("general.useragent.locale", get_language()) self.selenium = Firefox( firefox_profile=profile, executable_path='node_modules/geckodriver/geckodriver') self.selenium.maximize_window()
class Driver: def __init__(self, profile): self.profile = FirefoxProfile(profile) self.driver = webdriver.Firefox(self.profile) self.driver.maximize_window() wait = WebDriverWait(self.driver, 10) def __init__(self, profile, userAgent=''): self.profile = FirefoxProfile(profile) if (userAgent != ''): self.profile.set_preference("general.useragent.override", userAgent) self.driver = webdriver.Firefox(self.profile) self.driver.execute_script( "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})" ) self.driver.maximize_window() wait = WebDriverWait(self.driver, 10) def getDriver(self): return self.driver
def run_test(): profile = FirefoxProfile("/Users/andrasta/Desktop/InfoVis/project/FirefoxProfile") profile.set_preference("browser.download.folderList", 2) profile.set_preference("browser.download.dir", os.getcwd() + "/QB") profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv") profile.set_preference("browser.helperApps.neverAsk.openFile", "text/csv") driver = webdriver.Firefox(firefox_profile=profile) # Doesn't matter # self.driver.maximize_window() wait = WebDriverWait(driver, 5) driver.maximize_window() driver.get("http://www.pro-football-reference.com/players/B/BreeDr00/gamelog//") csvXPath = "(//span[contains(.,'Export')])[1]" csvLink = wait.until(EC.element_to_be_clickable((By.XPATH, csvXPath))) print "XPATH found link text is: ", csvLink.text if csvLink != None and csvLink.text == "Export": # ActionChains(driver).move_to_element(csvLink).perform() ActionChains(driver).click(csvLink).perform()
def setUp(self): from selenium.webdriver.firefox.webdriver import FirefoxProfile fp = FirefoxProfile() fp.set_preference("browser.download.folderList", 2) fp.set_preference("browser.download.dir", self.DOWNLOAD_DIRECTORY) fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/html, application/zip, text/plain, application/force-download, application/x-tar") self.selenium = WebDriver(firefox_profile=fp) self.selenium.implicitly_wait(1) # wait one second before failing to find # create the download dir if not os.path.exists(self.DOWNLOAD_DIRECTORY): os.makedirs(self.DOWNLOAD_DIRECTORY)
def setUp(self): self.output_formats = GlobalParameterFactory.create(parameter_name='output_formats', parameter_value=LiveServerTest.OUTPUT_FORMATS) from selenium.webdriver.firefox.webdriver import FirefoxProfile fp = FirefoxProfile() fp.set_preference("browser.download.folderList", 2) fp.set_preference("browser.download.dir", self.DOWNLOAD_DIRECTORY) fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/html, application/zip, text/plain, application/force-download, application/x-tar") self.selenium = WebDriver(firefox_profile=fp) self.selenium.implicitly_wait(1) # wait one second before failing to find # create the download dir if not os.path.exists(self.DOWNLOAD_DIRECTORY): os.makedirs(self.DOWNLOAD_DIRECTORY) # ensure that it is empty for root, dirs, files in os.walk(self.DOWNLOAD_DIRECTORY): for file in files: os.remove(os.path.join(root, file)) self.halt_on_exception = os.environ.get('TAO_HALT_ON_EXCEPTION', False) == 'True' return
# CHROME_OPTIONS.add_argument("touch-events") profile = FirefoxProfile() # profile.set_preference("webdriver.log.file", # "/tmp/firefox_webdriver.log") # profile.set_preference("webdriver.firefox.logfile", # "/tmp/firefox.log") # # This turns off the downloading prompt in FF. # tmp_path = "selenium_tests/tmp" shutil.rmtree(tmp_path, True) os.makedirs(tmp_path) profile.set_preference("browser.download.folderList", 2) profile.set_preference("browser.download.manager.showWhenStarting", False) profile.set_preference("browser.download.dir", tmp_path) profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/xml") FIREFOX_PROFILE = profile def post_execution(): shutil.rmtree(tmp_path, True) if CONFIG.remote and not REMOTE_SERVICE: raise ValueError("you must pass a service argument to behave") # # Location of our server. Changing this use standalone rather than packed will
# CHROME_OPTIONS.add_argument("touch-events") profile = FirefoxProfile() # profile.set_preference("webdriver.log.file", # "/tmp/firefox_webdriver.log") # profile.set_preference("webdriver.firefox.logfile", # "/tmp/firefox.log") # # This turns off the downloading prompt in FF. # tmp_path = "selenium_tests/tmp" shutil.rmtree(tmp_path, True) os.makedirs(tmp_path) profile.set_preference("browser.download.folderList", 2) profile.set_preference("browser.download.manager.showWhenStarting", False) profile.set_preference("browser.download.dir", tmp_path) profile.set_preference( "browser.helperApps.neverAsk.saveToDisk", "text/xml") FIREFOX_PROFILE = profile def post_execution(): shutil.rmtree(tmp_path, True) # May be required to get native events. # FIREFOX_BINARY = FirefoxBinary("/home/ldd/src/firefox-24/firefox") # # Location of our server. Changing this use standalone rather than
def __init__(self, input_file): # print "input file: " + input_file self.stage = 0 # use stage to see where in the process you are so you know where to come back to. self.whoops = ( 0 ) # number of mistakes on this particular's entry's upload. you can change number of mistakes allowed in webFailure function. self.volume = Volume() self.volume.travelAgent(input_file) # makevolume self.local = Local(self.volume) self.t_start = datetime.now() self.service_type = "//*[@id='productline_3']" self.book_size = "//*[@id='preset_1037_73']" self.binding = "//*[@id='binding_4']" self.pg_count = "//*[@id='pagecount']" self.uid = "//*[@id='loginEmail']" # id/name for logging in self.pw = "//*[@id='loginPassword']" # password area self.logbutt = "//*[@id='loginSubmit']" # submit button self.my_page = "700" self.next = "//*[@id='fNext']" self.next_disable = "//*[@id='fNext' and not (@disabled)]" self.book_title = "//*[@id='title']" # get and put title self.fname = "//*[@id='firstName']" self.lname = "//*[@id='lastName']" self.project_details = "/html/body/div[1]/div[1]/div/div/div[2]" self.project_id = "/html/body/div[1]/div[1]/div/div/div[2]/div[2]/table/tbody/tr[1]/td[1]" self.myFiles = "//*[@id='ui-id-2']" self.cover_upload = "//*[@id='fOnePieceCoverFile']" self.x_lulu_isbn = "//*[@id='projectInformationSection']/div[2]/table/tbody/tr[1]/td" self.x_lulu_id = "/html/body/div[1]/div[3]/div[3]/div[2]/div/div[2]/form/div[1]/div[2]/div/div[2]/div[2]/table/tbody/tr[1]/td" self.x_sell = "/html/body/div[1]/div[3]/div[2]/div[2]/div/div[1]/div/h3/a" self.sku_x = "/html/body/div[1]/div[2]/div[2]/div[1]/div[2]/div[2]/form/input[1]" self.bc_image = "/html/body/div/div[3]/div[2]/div[2]/div/div[2]/div[4]/a/img" self.folders_option = "//*[@id='labelFilter']" self.num_Xpath = "//*[@id='pageCount']" self.submit_button = "/html/body/div[1]/div[1]/div/table/tbody/tr/td[2]/div/div[1]/table/tbody/tr/td[1]/input" self.first_table = "//*[@id='pageLinks']/a[1]" self.one_file_cover = "/html/body/div[1]/div[5]/div/div[2]/a[2]" self.c_up_button = "//*[@id='fMegaUpload']" self.keywords = "//*[@id='keywords']" self.description = "//*[@id='descr']" self.copyright = "//*[@id='copyright']" self.license = "//*[@id='license']" self.edition = "//*[@id='edition']" self.publisher = "//*[@id='publisher']" self.setPrice = "//*[@id='userPrice']" self.x_lulu_isbn = "//*[@id='projectInformationSection']/div[2]/table/tbody/tr[1]/td" self.x_sell = "/html/body/div[1]/div[3]/div[2]/div[2]/div/div[1]/div/h3/a" self.sku_x = "/html/body/div[1]/div[2]/div[2]/div[1]/div[2]/div[2]/form/input[1]" self.helv_in_enc_file = "/html/body/div[2]/div[1]/div/div[1]/div/div" profile = FirefoxProfile( "/Users/wiki/Library/Application Support/Firefox/Profiles/3ig8qcrl.default-1435148476151" ) # profile is necessary to run the greasemonkey scripts. profile.set_preference("toolkit.startup.max_resumed_crashes", "-1") self.driver = webdriver.Firefox(profile) # start ff w/profile self.driver.set_window_position(6920, 0) # browser size and position. self.driver.set_window_size(220, 220) print "opening new browser to automate upload of: " + self.volume.title + " on Firefox" self.driver.get("http://www.lulu.com/author/wizard/index.php?fWizard=hardcover") # navigate to author page print "cur time= " + str(self.t_start) self.pdf_pages = self.count_pages(self.local.inFolder + self.volume.round_folder + "/" + input_file) self.pdf_pages = str(self.pdf_pages) self.luluCruise() # start!
import sqlite3 import time import re from selenium import webdriver from selenium.webdriver.firefox.firefox_binary import FirefoxBinary from selenium.webdriver.firefox.webdriver import FirefoxProfile profile = FirefoxProfile("C:\\Users\\gokha\\AppData\\Roaming\\Mozilla\\Firefox\\Profiles\\b21rnkyt.kariyer") profile.add_extension(r'C:\\Users\\gokha\\AppData\\Roaming\\Mozilla\\Firefox\\Profiles\\b21rnkyt.kariyer\\extensions\\adblock.xpi') # for adblockplus profile.set_preference("extensions.adblockplus.currentVersion","3.7") binary = FirefoxBinary("C:\\Program Files\\Mozilla Firefox\\firefox.exe") browser = webdriver.Firefox(firefox_profile=profile, firefox_binary=binary) browser.maximize_window() sayfano = 0 linkler = [] durum = "0" conn = sqlite3.connect('veritaban.db') c = conn.cursor() def create_table(): c.execute('CREATE TABLE IF NOT EXISTS basvurulinkleri(urll TEXT,durumu TEXT)') def giris_yap(): kullanıcı_adı = input() sifre = input() browser.get("https://www.kariyer.net/website/kariyerim/login.aspx") time.sleep(3) username = browser.find_element_by_xpath("//*[@id='lgnUserName']") password = browser.find_element_by_xpath("//*[@id='lgnPassword']") username.send_keys(kullanıcı_adı) password.send_keys(sifre)
def get_profile(self): profile = FirefoxProfile(self.profile_dir) profile.set_preference('startup.homepage_welcome_url', "about:blank") profile.set_preference('browser.startup.homepage', "about:blank") # profile.set_preference('extensions.firebug.netexport.defaultLogDir', self.netlog_dir) #set socks proxy profile.set_preference("network.proxy.type", 1) profile.set_preference("network.proxy.socks_version", 5) profile.set_preference("network.proxy.socks", '127.0.0.1') profile.set_preference("network.proxy.socks_port", 9150) profile.set_preference("network.proxy.socks_remote_dns", True) profile.set_preference("places.history.enabled", False) profile.set_preference("privacy.clearOnShutdown.offlineApps", True) profile.set_preference("privacy.clearOnShutdown.passwords", True) profile.set_preference("privacy.clearOnShutdown.siteSettings", True) profile.set_preference("privacy.sanitize.sanitizeOnShutdown", True) profile.set_preference("signon.rememberSignons", False) profile.set_preference("network.dns.disablePrefetch", True) profile.set_preference( "extensions.firebug.netexport.pageLoadedTimeout", 10000) profile.set_preference("extensions.firebug.netexport.showPreview", False) profile.set_preference( "extensions.firebug.netexport.alwaysEnableAutoExport", False) profile.set_preference("extensions.firebug.DBG_STARTER", False) profile.set_preference("extensions.firebug.onByDefault", False) profile.set_preference("extensions.firebug.allPagesActivation", "off") profile.update_preferences() return profile
def use_browser(self): self.use_selenium = True firefox_profile = FirefoxProfile() # might as well turn off images since we don't need them if self.use_proxies: # if use proxies is true load firefox with proxies firefox_profile.set_preference("permissions.default.image", 2) proxy_host, proxy_port = choice(self.proxies).split(':') firefox_profile.set_preference("network.proxy.type", 1) firefox_profile.set_preference("network.proxy.http", proxy_host) firefox_profile.set_preference("network.proxy.http_port", int(proxy_port)) firefox_profile.set_preference("network.proxy.ssl", proxy_host) firefox_profile.set_preference("network.proxy.ssl_port", int(proxy_port)) self.driver = Firefox(firefox_profile) self.driver.implicitly_wait(2)
def get_profile(self): profile = FirefoxProfile(self.profile_dir) profile.set_preference('startup.homepage_welcome_url', "about:blank") profile.set_preference('browser.startup.homepage', "about:blank") # profile.set_preference('extensions.firebug.netexport.defaultLogDir', self.netlog_dir) #set socks proxy profile.set_preference( "network.proxy.type", 1 ) profile.set_preference( "network.proxy.socks_version", 5 ) profile.set_preference( "network.proxy.socks", '127.0.0.1' ) profile.set_preference( "network.proxy.socks_port", 9150) profile.set_preference( "network.proxy.socks_remote_dns", True ) profile.set_preference( "places.history.enabled", False ) profile.set_preference( "privacy.clearOnShutdown.offlineApps", True ) profile.set_preference( "privacy.clearOnShutdown.passwords", True ) profile.set_preference( "privacy.clearOnShutdown.siteSettings", True ) profile.set_preference( "privacy.sanitize.sanitizeOnShutdown", True ) profile.set_preference( "signon.rememberSignons", False ) profile.set_preference( "network.dns.disablePrefetch", True ) profile.set_preference("extensions.firebug.netexport.pageLoadedTimeout", 10000) profile.set_preference("extensions.firebug.netexport.showPreview", False) profile.set_preference("extensions.firebug.netexport.alwaysEnableAutoExport", False) profile.set_preference("extensions.firebug.DBG_STARTER", False) profile.set_preference("extensions.firebug.onByDefault", False) profile.set_preference("extensions.firebug.allPagesActivation", "off") profile.update_preferences() return profile
def __init__(self, IPass, Volume, Local): self.const = IPass if self.const.make_log is True: #send output to logfile. log_file = open("log.txt","w") sys.stdout = log_file # print "input file: " + input_file self.stage = 0 #use stage to see where in the process you are so you know where to come back to. self.whoops = 0 #number of mistakes on this particular's entry's upload. you can change number of mistakes allowed in webFailure function. self.volume = Volume self.volume.travelAgent(self.const.input_file) #makevolume self.local = Local self.t_start = datetime.now() self.service_type = "//*[@id='productline_3']" self.service_type2 = "//*[@id='productline_3']/img" self.book_size = "//*[@id='preset_1040_0']" self.book_size2 = "//*[@id='preset_1040_0']/img" self.titlexpath = "/html/body/div[1]/div[3]/div[2]/div[1]/div[1]/h2" self.binding = "//*[@id='binding_4']" self.pg_count = "//*[@id='pagecount']" self.uid = "//*[@id='loginEmail']" #id/name for logging in self.pw = "//*[@id='loginPassword']" #password area self.logbutt = "//*[@id='loginSubmit']" #submit button self.my_page = "700" self.next = "//*[@id='fNext']" self.next_disable = "//*[@id='fNext' and not (@disabled)]" self.book_title = "//*[@id='title']"#get and put title self.fname = "//*[@id='firstName']" self.lname = "//*[@id='lastName']" self.project_details = "/html/body/div[1]/div[1]/div/div/div[2]" self.project_id = "/html/body/div[1]/div[1]/div/div/div[2]/div[2]/table/tbody/tr[1]/td[1]" self.myFiles = "//*[@id='ui-id-2']" self.cover_upload = "//*[@id='fOnePieceCoverFile']" self.x_lulu_isbn = "//*[@id='projectInformationSection']/div[2]/table/tbody/tr[1]/td" self.x_lulu_id = "/html/body/div[1]/div[3]/div[3]/div[2]/div/div[2]/form/div[1]/div[2]/div/div[2]/div[2]/table/tbody/tr[1]/td" self.x_sell = "/html/body/div[1]/div[3]/div[2]/div[2]/div/div[1]/div/h3/a" self.sku_x = "/html/body/div[1]/div[2]/div[2]/div[1]/div[2]/div[2]/form/input[1]" self.bc_image = "/html/body/div/div[3]/div[2]/div[2]/div/div[2]/div[4]/a/img" self.folders_option = "//*[@id='labelFilter']" self.first_checkbox = "/html/body/div[1]/div[1]/div/table/tbody/tr/td[2]/div/div[3]/div[1]/form/div/div/div[2]/table/tbody/tr[1]/td[1]/input" #always the first input since it was most recently uploaded. no need to iterate. self.num_Xpath = "//*[@id='pageCount']" self.submit_button = "/html/body/div[1]/div[1]/div/table/tbody/tr/td[2]/div/div[1]/table/tbody/tr/td[1]/input" self.first_table = "//*[@id='pageLinks']/a[1]" self.one_file_cover = "/html/body/div[1]/div[5]/div/div[2]/a[2]" self.c_up_button = "//*[@id='fMegaUpload']" self.keywords = "//*[@id='keywords']" self.description = "//*[@id='descr']" self.copyright = "//*[@id='copyright']" self.license = "//*[@id='license']" self.edition = "//*[@id='edition']" self.publisher = "//*[@id='publisher']" self.setPrice = "//*[@id='userPrice']" self.x_lulu_isbn = "//*[@id='projectInformationSection']/div[2]/table/tbody/tr[1]/td" self.x_sell = "/html/body/div[1]/div[3]/div[2]/div[2]/div/div[1]/div/h3/a" self.sku_x = "/html/body/div[1]/div[2]/div[2]/div[1]/div[2]/div[2]/form/input[1]" self.helv_in_enc_file = "/html/body/div[2]/div[1]/div/div[1]/div/div" profile = FirefoxProfile("/Users/wiki/Library/Application Support/Firefox/Profiles/3ig8qcrl.default-1435148476151")#profile is necessary to run the greasemonkey scripts. profile.set_preference("toolkit.startup.max_resumed_crashes", "-1") self.private_input = "/html/body/div[1]/div[3]/div[3]/div[2]/div/div[2]/form/div[1]/div[2]/div/div[4]/div[2]/ul/li[1]/input" self.driver = webdriver.Firefox(profile) #start ff w/profile self.driver.set_window_position(6920,0)#browser size and position. self.driver.set_window_size(3460,1120) print "opening new browser to automate upload of: " + self.volume.title + " on Firefox" self.driver.get("http://www.lulu.com/author/wizard/index.php?fWizard=hardcover")#navigate to author page print "cur time= " + str(self.t_start) self.pdf_pages = self.count_pages(self.local.inFolder+self.volume.round_folder+"/"+self.volume.input_file) self.pdf_pages=str(self.pdf_pages) self.luluCruise() #start!
def _set_profile(): useragent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' \ 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' profile = FirefoxProfile() profile.set_preference('general.useragent.override', useragent) return profile
def setup_test_environment(self, **kwargs): super(SeleniumTestSuiteRunner, self).setup_test_environment(**kwargs) profile = FirefoxProfile() profile.add_extension(settings.FIREBUG_EXTENSION) profile.add_extension(settings.FIRESTARTER_EXTENSION) profile.set_preference('extensions.firebug.currentVersion', '1.10.3') profile.set_preference('extensions.firebug.allPagesActivation', 'on') profile.set_preference('extensions.firebug.previousPlacement', 1) profile.set_preference('extensions.firebug.onByDefault', True) profile.set_preference('extensions.firebug.showFirstRunPage', False) profile.set_preference('extensions.firebug.defaultPanelName', 'console') profile.set_preference('extensions.firebug.net.enableSites', True) profile.set_preference('extensions.firebug.console.enableSites', True) self.__class__.selenium = WebDriver(firefox_profile=profile)