def get_basic_setup() -> BrowserSetup: browser_setup = BrowserSetup() if properties.OS_LINUX in platform: print(properties.OS_LINUX) options = Options() options.accept_untrusted_certs = True options.assume_untrusted_cert_issuer = True browser_setup.options = options # browser_setup.web_home_path = "/static/index.html" browser_setup.driver_path = properties.DRIVER_LINUX_PATH elif properties.OS_MAC in platform: print(properties.OS_MAC) options = Options() options.accept_untrusted_certs = True options.assume_untrusted_cert_issuer = True browser_setup.options = options # browser_setup.web_home_path = "/static/index.html" browser_setup.driver_path = properties.DRIVER_MAC_PATH elif properties.OS_WINDOW in platform: print(properties.OS_WINDOW) options = Options() options.accept_untrusted_certs = True options.assume_untrusted_cert_issuer = True options.add_argument(properties.USER_DATA_DIR) browser_setup.options = options # browser_setup.web_home_path = "\\static\\index.html" browser_setup.driver_path = properties.DRIVER_WINDOWS_PATH else: raise UnsupportedPlatform return browser_setup
def stat_scraper(link_list, driver): #truncate_table(table_name) options = Options() options.accept_untrusted_certs = True options.assume_untrusted_cert_issuer = True options.add_argument('--load-extension=/Users/Philip/Documents/NBA prediction script/Incremental Pipelines/3.34.0_0') browser = webdriver.Chrome(executable_path=driver, chrome_options=options) browser.get(link_list[0]) #browser.find_element_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/div[1]/div[1]/div/div/label/select/option[1]').click() try: wait = WebDriverWait(browser, 120).until(EC.presence_of_element_located((By.XPATH, '/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select'))) except TimeoutException or NoSuchElementException: browser.refresh() #logging.info('Failed to connect to page') pages = browser.find_element_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select') options = [x for x in pages.find_elements_by_tag_name("option")] for page in range(2, len(options)+1): browser.find_element_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[3]/div/div/select/option[{}]'.format(str(page))).click() table = browser.find_element_by_class_name('nba-stat-table') content = table.get_attribute('innerHTML') df = pd.read_html(content)[0] f_df = format_matchup(df) f_df['Game Date'] = f_df.loc[:, 'Game Date'].apply(convert_date) insert_into_database(f_df, link_list[1]) browser.quit() return
def gen_browser(link, driver): options = Options() options.accept_untrusted_certs = True options.assume_untrusted_cert_issuer = True options.add_argument('--load-extension=/Users/Philip/Documents/NBA prediction script/Incremental Pipelines/3.34.0_0') browser = webdriver.Chrome(executable_path=driver, chrome_options=options) browser.get(link) try: wait = WebDriverWait(browser, 120).until(EC.presence_of_element_located((By.XPATH, '/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select'))) except TimeoutException or NoSuchElementException: browser.refresh() logging.info('[FAILED TO CONNECT TO PAGE]: {}'.format(link)) return browser
def do_screenshots_of_target_hosts(): show_time("starting screenshoting targets") options = Options() options.headless = True options.accept_untrusted_certs = True options.assume_untrusted_cert_issuer = True options.add_argument("--no-sandbox") options.add_argument('--disable-dev-shm-usage') driver = webdriver.Chrome(options=options) os.makedirs(args.domain, exist_ok=True) for key, value in ip_addresses.items(): if 80 in value: if value[80]['has_content']: execute_browser('http://', key, value[80]['start_url'], driver, value['source'], value[80]['http_status_code']) if 443 in value: if value[443]['has_content']: execute_browser('https://', key, value[443]['start_url'], driver, value['source'], value[443]['http_status_code']) driver.close() driver.quit() show_time("screenshoting targets finished")
def run(self): try: import os driverName = "\\chromedriver.exe" # defining base file directory of chrome drivers driver_loc = os.path.dirname( os.path.abspath(__file__)) + "\\ChromeDriver\\" # defining the file path of your exe file automatically updating based on your browsers current version of chrome. currentPath = driver_loc + chrome_browser_version + driverName # check if new version of drive exists --> only continue if it doesn't Newpath = driver_loc + nextVersion # check if we have already downloaded the newest version of the browser newfileloc = Newpath + driverName newpathexists = os.path.exists(newfileloc) if newpathexists == False: try: # open chrome driver and attempt to download new chrome driver exe file. # set the arguments and options chromeOptions = Options() chromeOptions.add_experimental_option( "prefs", { "download.default_directory": driver_loc, "download.prompt_for_download": False, "download.directory_upgrade": True, "safebrowsing.enabled": True, "profile.managed_default_content_settings.images": 2, }, ) chromeOptions.add_experimental_option( "excludeSwitches", ["enable-logging"]) chromeOptions.add_argument("--headless") chromeOptions.add_argument( "--blink-settings=imagesEnabled=false") chromeOptions.add_argument("--disable-popup-blocking") chromeOptions.add_argument("--ignore-certificate-errors") chromeOptions.add_argument("--allow-insecure-localhost") chromeOptions.add_argument( "--allow-running-insecure-content") chromeOptions.accept_untrusted_certs = True chromeOptions.assume_untrusted_cert_issuer = True service_args = ["hide_console"] try: print("~~~Calling Update Driver") update_driver = Chrome( executable_path=currentPath, options=chromeOptions, service_args=service_args, ) print("~~~Update Driver Opened") # opening up url of chromedriver to get new version of chromedriver. chromeDriverURL = ( "https://chromedriver.storage.googleapis.com/index.html?path=" + nextVersion) update_driver.set_page_load_timeout(10) update_driver.delete_all_cookies() update_driver.get(chromeDriverURL) print("~~~Update Website Got") # time.sleep(5) event.wait(5) # find records of table rows table = update_driver.find_elements_by_css_selector( "tr") # check the length of the table Table_len = len(table) # ensure that table length is greater than 4, else fail. -- table length of 4 is default when there are no availble updates if Table_len > 4: # define string value of link rowText = table[(len(table) - 2)].text[:6] # time.sleep(1) event.wait(1) # select the value of the row update_driver.find_element_by_xpath( "//*[contains(text()," + '"' + str(rowText) + '"' + ")]").click() event.wait(1) # time.sleep(1) # select chromedriver zip for windows update_driver.find_element_by_xpath( "//*[contains(text()," + '"' + "win32" + '"' + ")]").click() print("~~~Download Started") # time.sleep(5) event.wait(5) update_driver.quit() print("~~~Update Driver Exited") try: from zipfile import ZipFile import shutil fileName = os.path.join( os.path.dirname(driver_loc), "chromedriver_win32.zip", ) # Create a ZipFile Object and load sample.zip in it with ZipFile(fileName, "r") as zipObj: # Extract all the contents of zip file in different directory zipObj.extractall(Newpath) print("~~~Newer Version Extracted") except Exception as ex: print( "Error in extracting:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) try: # delete downloaded file os.remove(fileName) print("Downloaded Zip Deleted") except Exception as ex: print( "~~~Error in deleting zip:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) # defining old chrome driver location oldPath = driver_loc + lastVersion oldpathexists = os.path.exists(oldPath) # this deletes the old folder with the older version of chromedriver in it if oldpathexists == True: try: import stat shutil.rmtree(oldPath, ignore_errors=True) print("Old Version Deleted") except Exception as ex: print( "~~~Error in deleting previous version:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) else: # update_driver.quit() print("~~~No new version available") except Exception as ex: print( "~~~Error in update driver:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) finally: try: # close the driver update_driver.quit() print("~~~Update Driver Exited") except Exception as ex: print( "~~~Error in quitting:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) except Exception as ex: print( "~~~Error in if:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) else: print("~~~ChromeDriver Upto Date~~~") except Exception as ex: print( "~~~Error in update:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args))
def run(self): """ Will check wheather the portal in logged in after a set interval and will sign in if not. """ try: # set the arguments and options chromeOptions = Options() prefs = {"profile.managed_default_content_settings.images": 2} chromeOptions.add_experimental_option("prefs", prefs) chromeOptions.add_experimental_option("excludeSwitches", ["enable-logging"]) chromeOptions.add_argument("--headless") chromeOptions.add_argument("--blink-settings=imagesEnabled=false") chromeOptions.add_argument("--disable-popup-blocking") chromeOptions.add_argument("--ignore-certificate-errors") chromeOptions.add_argument("--allow-insecure-localhost") chromeOptions.add_argument("--allow-running-insecure-content") chromeOptions.accept_untrusted_certs = True chromeOptions.assume_untrusted_cert_issuer = True service_args = ["hide_console"] currentPath = (os.path.dirname(os.path.abspath(__file__)) + "\\ChromeDriver\\" + chrome_browser_version + "\\chromedriver.exe") while True: try: print("\nCalling Driver") # Creating an instance of chrome driver = Chrome( executable_path=currentPath, options=chromeOptions, service_args=service_args, ) print("Driver Called") driver.set_page_load_timeout(10) driver.delete_all_cookies() # open a page driver.get("Enter Checking Site Here") print("Getting Site") try: """ remove this try except if the your wifi doesn't block websites """ # xpath if the website is blocked element = driver.find_element_by_xpath( "Enter xpath to an element in the blocked page") print("Site Blocked\n") except: try: # xpath to any thing in the website to make sure you are connected to the internet element = driver.find_element_by_xpath( "/Enter xpath to an element in the page") print("Site Opening\n") except: try: """ if your portal doesn't have auto redirect, uncomment the following line and type in your login url """ # driver.get("Paste Login Webiste URL Here") # change the ids to those in your login website # you can use developer mode to find the id of fields (use ctrl + shift + i) # change the username and password to the required one print("Trying To Login") # select usnername field element = driver.find_element_by_id( "Ending id of user input field") print("User Found") element.send_keys("Enter username") print("User Inputted") # select password field element = driver.find_element_by_id( "Ending id of password input field") print("Passwprd Found") element.send_keys("Enter password") print("Password Inputted") # select submit button element = driver.find_element_by_id( "Enter id of submit button") print("Button Found") element.click() print("Logged In\n") # except NoSuchElementException as ex: # print("Can't Login") # event.wait(120) except Exception as ex: print( "Can't login:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) event.wait(60) continue except Exception as ex: print( "Error in loop:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) try: driver.quit() except Exception as ex: print( "Error in Quitting:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) event.wait(60) continue try: driver.quit() except Exception as ex: print( "Error in Quitting in loop:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) event.wait(300) continue except Exception as ex: print( "Error outside loop:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) finally: try: driver.quit() except Exception as ex: print( "Error in Quitting in final:\t\tAn exception of type {0} occurred. Arguments:\n{1}" .format(type(ex).__name__, ex.args)) finally: print("Login Thread Exited")
def login(email): global valid global invalid global count colors = { 'merah': Fore.RED, 'hijau': Fore.GREEN, 'putih': Fore.WHITE, 'kuning': Fore.YELLOW, 'biru': Fore.CYAN, 'black': Fore.BLACK } if os.system == "nt": path = "chromedriver.exe" else: path = "chromedriver" url = "https://www.remitly.com/us/en/users/register" f = open("proxies.txt", "r") proxies = f for proxi in proxies: myproxy = proxi prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = myproxy prox.socks5_proxy = myproxy prox.ssl_proxy = myproxy capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) options = Options() options.headless = True options.add_argument('log-level=3') options.accept_untrusted_certs = True options.assume_untrusted_cert_issuer = True options.add_experimental_option('excludeSwitches', ['enable-logging']) options.add_argument("--no-sandbox") options.add_argument("--disable-impl-side-painting") options.add_argument('--disable-gpu') options.add_argument("--disable-setuid-sandbox") options.add_argument("--disable-seccomp-filter-sandbox") options.add_argument("--disable-breakpad") options.add_argument("--disable-client-side-phishing-detection") options.add_argument("--disable-cast") options.add_argument("--disable-cast-streaming-hw-encoding") options.add_argument("--disable-cloud-import") options.add_argument("--disable-popup-blocking") options.add_argument("--ignore-certificate-errors") options.add_argument("--disable-session-crashed-bubble") options.add_argument("--disable-ipv6") options.add_argument("--allow-http-screen-capture") options.add_argument("--start-maximized") driver = webdriver.Chrome(path, chrome_options=options, desired_capabilities=capabilities) driver.delete_all_cookies() driver.get(url) driver.find_element_by_name('email_address').send_keys(email) driver.find_element_by_name('password').send_keys('jakarta321') Select( driver.find_element_by_xpath( '/html/body/div[2]/div[2]/div/div/form/div[3]/div/select') ).select_by_value('4') button = driver.find_element_by_xpath( '/html/body/div[2]/div[2]/div/div/form/div[4]/button') button.click() # response = driver.find_elements_by_xpath("/html/body/div[2]/div[2]/div/div/script[1]") sleep(1) response = driver.execute_script( "return document.getElementsByTagName('html')[0].innerHTML") if 'This email address is associated with another account.' in ( str(response)): with myLock: Pac = email loglive.write(Pac) count = count + 1 valid = valid + 1 print("LIVE => " + email) else: with myLock: logdie.write(email) count = count + 1 invalid = invalid + 1 print("DIE => " + email)
def initialize_driver(headless=True, with_ublock=False, chromedriver_path=None): if not chromedriver_path: try: chromedriver_path = chromedriver_autoinstaller.install() except Exception as exception: log.critical( f"Failed to install the built-in chromedriver: {exception}\n" "download the correct version for your system at " "https://chromedriver.chromium.org/downloads and use the" "--chromedriver argument to point to the chromedriver executable" ) sys.exit() log.info(f"Initialising chromedriver at {chromedriver_path}...") chrome_options = Options() chrome_options.add_argument( "--disable-blink-features=AutomationControlled") chrome_options.add_experimental_option( "excludeSwitches", ["enable-automation", "enable-logging"]) chrome_options.add_experimental_option('useAutomationExtension', False) chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.accept_untrusted_certs = True chrome_options.assume_untrusted_cert_issuer = True chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--allow-http-screen-capture") chrome_options.add_argument("--disable-impl-side-painting") chrome_options.add_argument("--disable-setuid-sandbox") chrome_options.add_argument("--disable-seccomp-filter-sandbox") if headless: chrome_options.add_argument("--headless") chrome_options.add_argument("window-size=1920,1080") chrome_options.add_argument("--log-level=3") chrome_options.add_argument("--silent") chrome_options.add_argument("--disable-logging") # allows selenium to accept cookies with a non-int64 'expiry' value chrome_options.add_experimental_option("w3c", False) # removes the 'DevTools listening' log message chrome_options.add_experimental_option("excludeSwitches", ["enable-logging"]) if with_ublock: chrome_options.add_extension( os.path.join(os.getcwd(), "bin", "ublock", "ublock-extension.crx")) logs_path = os.path.join(os.getcwd(), "logs") if not (os.path.isdir(logs_path)): os.makedirs(logs_path) driver = webdriver.Chrome( executable_path=chromedriver_path, service_log_path=os.path.join(logs_path, "webdrive.log"), # Don't verify self-signed cert, should help with 502 errors # (https://github.com/wkeeling/selenium-wire/issues/55) # seleniumwire_options={"verify_ssl": False}, options=chrome_options, ) if with_ublock: log.debug("Configuring uBlock") # set up uBlock driver.get( "chrome-extension://ilchdfhfciidacichehpmmjclkbfaecg/settings.html" ) # Un-hide the file upload button so we can use it element = driver.find_elements_by_class_name("hidden") driver.execute_script( "document.getElementsByClassName('hidden')[0].className = ''", element) # scroll down (for debugging) driver.execute_script("window.scrollTo(0, 2000)") uBlock_settings_file = str( os.path.join(os.getcwd(), "bin", "ublock", "ublock-settings.txt")) driver.find_element_by_id("restoreFilePicker").send_keys( uBlock_settings_file) try: WebDriverWait(driver, 3).until(EC.alert_is_present()) # click ok on pop up to accept overwrite driver.switch_to.alert.accept() except TimeoutException as ex: log.error("Timeout waiting for ublock config overwrite alert") # leave uBlock config driver.get("about:blank") return driver