Пример #1
0
def get_basic_setup() -> BrowserSetup:
    browser_setup = BrowserSetup()
    if properties.OS_LINUX in platform:
        print(properties.OS_LINUX)
        options = Options()
        options.accept_untrusted_certs = True
        options.assume_untrusted_cert_issuer = True
        browser_setup.options = options
        # browser_setup.web_home_path = "/static/index.html"
        browser_setup.driver_path = properties.DRIVER_LINUX_PATH
    elif properties.OS_MAC in platform:
        print(properties.OS_MAC)
        options = Options()
        options.accept_untrusted_certs = True
        options.assume_untrusted_cert_issuer = True
        browser_setup.options = options
        # browser_setup.web_home_path = "/static/index.html"
        browser_setup.driver_path = properties.DRIVER_MAC_PATH
    elif properties.OS_WINDOW in platform:
        print(properties.OS_WINDOW)
        options = Options()
        options.accept_untrusted_certs = True
        options.assume_untrusted_cert_issuer = True
        options.add_argument(properties.USER_DATA_DIR)
        browser_setup.options = options
        # browser_setup.web_home_path = "\\static\\index.html"
        browser_setup.driver_path = properties.DRIVER_WINDOWS_PATH
    else:
        raise UnsupportedPlatform

    return browser_setup
Пример #2
0
    def login(self, ip=None, password=None):
        self.baseurl = "https://" + ip

        options = Options()
        options.set_capability("acceptInsecureCerts", True)
        options.accept_untrusted_certs = True
        options.add_argument('--ignore-certificate-errors')
        if "systest-dev1" == gethostname():
            options.headless = True
        now = datetime.now()
        now = now.strftime("%d-%b-%Y-%H:%M:%S.%f")
        self.wd = webdriver.Chrome(options=options,
                                   service_args=[
                                       "--verbose", "--log-path=/tmp/" + now +
                                       " chromedriver.log"
                                   ])
        self.wd.set_window_size(1366, 758)
        self.wd.implicitly_wait(10)
        time.sleep(1)
        url = self.baseurl + "#/admin"
        self.wd.get(url)
        element = self.wd.find_element_by_id("admin-tab")
        element.click()

        time.sleep(1)

        element = self.wd.find_element_by_id("passwordInput")
        element.clear()
        element.send_keys(password)

        time.sleep(1)
        element = self.wd.find_element_by_id("passEnter")
        self.wd.execute_script("document.getElementById('passEnter').click()")
        # not so fast...
        time.sleep(2)
Пример #3
0
def stat_scraper(link_list, driver):
    #truncate_table(table_name)
    options = Options()
    options.accept_untrusted_certs = True
    options.assume_untrusted_cert_issuer = True
    options.add_argument('--load-extension=/Users/Philip/Documents/NBA prediction script/Incremental Pipelines/3.34.0_0')
    browser = webdriver.Chrome(executable_path=driver, chrome_options=options)
    browser.get(link_list[0])
    #browser.find_element_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/div[1]/div[1]/div/div/label/select/option[1]').click()

    try:
        wait = WebDriverWait(browser, 120).until(EC.presence_of_element_located((By.XPATH, '/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select')))
    except TimeoutException or NoSuchElementException:
        browser.refresh()
        #logging.info('Failed to connect to page')

    pages = browser.find_element_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select')
    options = [x for x in pages.find_elements_by_tag_name("option")]

    for page in range(2, len(options)+1):
        browser.find_element_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[3]/div/div/select/option[{}]'.format(str(page))).click()
        table = browser.find_element_by_class_name('nba-stat-table')
        content = table.get_attribute('innerHTML')
        df = pd.read_html(content)[0]
        f_df = format_matchup(df)
        f_df['Game Date']  = f_df.loc[:, 'Game Date'].apply(convert_date)
        insert_into_database(f_df, link_list[1])
    browser.quit()
    return
def gen_browser(link, driver):
    options = Options()
    options.accept_untrusted_certs = True
    options.assume_untrusted_cert_issuer = True
    options.add_argument('--load-extension=/Users/Philip/Documents/NBA prediction script/Incremental Pipelines/3.34.0_0')
    browser = webdriver.Chrome(executable_path=driver, chrome_options=options)
    browser.get(link)

    try:
        wait = WebDriverWait(browser, 120).until(EC.presence_of_element_located((By.XPATH, '/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select')))
    except TimeoutException or NoSuchElementException:
        browser.refresh()
        logging.info('[FAILED TO CONNECT TO PAGE]: {}'.format(link))
    return browser
Пример #5
0
def do_screenshots_of_target_hosts():
    show_time("starting screenshoting targets")
    options = Options()
    options.headless = True
    options.accept_untrusted_certs = True
    options.assume_untrusted_cert_issuer = True
    options.add_argument("--no-sandbox")
    options.add_argument('--disable-dev-shm-usage')
    driver = webdriver.Chrome(options=options)
    os.makedirs(args.domain, exist_ok=True)
    for key, value in ip_addresses.items():
        if 80 in value:
            if value[80]['has_content']:
                execute_browser('http://', key, value[80]['start_url'], driver,
                                value['source'], value[80]['http_status_code'])
        if 443 in value:
            if value[443]['has_content']:
                execute_browser('https://', key, value[443]['start_url'],
                                driver, value['source'],
                                value[443]['http_status_code'])
    driver.close()
    driver.quit()
    show_time("screenshoting targets finished")
Пример #6
0
    def run(self):
        try:
            import os

            driverName = "\\chromedriver.exe"

            # defining base file directory of chrome drivers
            driver_loc = os.path.dirname(
                os.path.abspath(__file__)) + "\\ChromeDriver\\"

            # defining the file path of your exe file automatically updating based on your browsers current version of chrome.
            currentPath = driver_loc + chrome_browser_version + driverName

            # check if new version of drive exists --> only continue if it doesn't
            Newpath = driver_loc + nextVersion

            # check if we have already downloaded the newest version of the browser
            newfileloc = Newpath + driverName
            newpathexists = os.path.exists(newfileloc)

            if newpathexists == False:
                try:
                    # open chrome driver and attempt to download new chrome driver exe file.

                    # set the arguments and options
                    chromeOptions = Options()
                    chromeOptions.add_experimental_option(
                        "prefs",
                        {
                            "download.default_directory": driver_loc,
                            "download.prompt_for_download": False,
                            "download.directory_upgrade": True,
                            "safebrowsing.enabled": True,
                            "profile.managed_default_content_settings.images":
                            2,
                        },
                    )
                    chromeOptions.add_experimental_option(
                        "excludeSwitches", ["enable-logging"])
                    chromeOptions.add_argument("--headless")
                    chromeOptions.add_argument(
                        "--blink-settings=imagesEnabled=false")
                    chromeOptions.add_argument("--disable-popup-blocking")
                    chromeOptions.add_argument("--ignore-certificate-errors")
                    chromeOptions.add_argument("--allow-insecure-localhost")
                    chromeOptions.add_argument(
                        "--allow-running-insecure-content")
                    chromeOptions.accept_untrusted_certs = True
                    chromeOptions.assume_untrusted_cert_issuer = True
                    service_args = ["hide_console"]

                    try:
                        print("~~~Calling Update Driver")
                        update_driver = Chrome(
                            executable_path=currentPath,
                            options=chromeOptions,
                            service_args=service_args,
                        )
                        print("~~~Update Driver Opened")

                        # opening up url of chromedriver to get new version of chromedriver.
                        chromeDriverURL = (
                            "https://chromedriver.storage.googleapis.com/index.html?path="
                            + nextVersion)
                        update_driver.set_page_load_timeout(10)
                        update_driver.delete_all_cookies()
                        update_driver.get(chromeDriverURL)
                        print("~~~Update Website Got")

                        # time.sleep(5)
                        event.wait(5)
                        # find records of table rows
                        table = update_driver.find_elements_by_css_selector(
                            "tr")

                        # check the length of the table
                        Table_len = len(table)

                        # ensure that table length is greater than 4, else fail. -- table length of 4 is default when there are no availble updates
                        if Table_len > 4:

                            # define string value of link
                            rowText = table[(len(table) - 2)].text[:6]
                            # time.sleep(1)
                            event.wait(1)

                            # select the value of the row
                            update_driver.find_element_by_xpath(
                                "//*[contains(text()," + '"' + str(rowText) +
                                '"' + ")]").click()
                            event.wait(1)
                            # time.sleep(1)

                            # select chromedriver zip for windows
                            update_driver.find_element_by_xpath(
                                "//*[contains(text()," + '"' + "win32" + '"' +
                                ")]").click()
                            print("~~~Download Started")

                            # time.sleep(5)
                            event.wait(5)
                            update_driver.quit()

                            print("~~~Update Driver Exited")

                            try:
                                from zipfile import ZipFile
                                import shutil

                                fileName = os.path.join(
                                    os.path.dirname(driver_loc),
                                    "chromedriver_win32.zip",
                                )

                                # Create a ZipFile Object and load sample.zip in it
                                with ZipFile(fileName, "r") as zipObj:
                                    # Extract all the contents of zip file in different directory
                                    zipObj.extractall(Newpath)

                                print("~~~Newer Version Extracted")
                            except Exception as ex:
                                print(
                                    "Error in extracting:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                                    .format(type(ex).__name__, ex.args))

                            try:
                                # delete downloaded file
                                os.remove(fileName)
                                print("Downloaded Zip Deleted")
                            except Exception as ex:
                                print(
                                    "~~~Error in deleting zip:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                                    .format(type(ex).__name__, ex.args))

                            # defining old chrome driver location
                            oldPath = driver_loc + lastVersion
                            oldpathexists = os.path.exists(oldPath)

                            # this deletes the old folder with the older version of chromedriver in it
                            if oldpathexists == True:
                                try:
                                    import stat

                                    shutil.rmtree(oldPath, ignore_errors=True)

                                    print("Old Version Deleted")
                                except Exception as ex:
                                    print(
                                        "~~~Error in deleting previous version:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                                        .format(type(ex).__name__, ex.args))

                        else:
                            # update_driver.quit()
                            print("~~~No new version available")

                    except Exception as ex:
                        print(
                            "~~~Error in update driver:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                            .format(type(ex).__name__, ex.args))

                    finally:
                        try:
                            # close the driver
                            update_driver.quit()
                            print("~~~Update Driver Exited")
                        except Exception as ex:
                            print(
                                "~~~Error in quitting:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                                .format(type(ex).__name__, ex.args))
                except Exception as ex:
                    print(
                        "~~~Error in if:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                        .format(type(ex).__name__, ex.args))
            else:
                print("~~~ChromeDriver Upto Date~~~")

        except Exception as ex:
            print(
                "~~~Error in update:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                .format(type(ex).__name__, ex.args))
Пример #7
0
    def run(self):
        """
            Will check wheather the portal in logged in after a set interval and will sign in if not. 
        """
        try:

            # set the arguments and options
            chromeOptions = Options()
            prefs = {"profile.managed_default_content_settings.images": 2}
            chromeOptions.add_experimental_option("prefs", prefs)
            chromeOptions.add_experimental_option("excludeSwitches",
                                                  ["enable-logging"])
            chromeOptions.add_argument("--headless")
            chromeOptions.add_argument("--blink-settings=imagesEnabled=false")
            chromeOptions.add_argument("--disable-popup-blocking")
            chromeOptions.add_argument("--ignore-certificate-errors")
            chromeOptions.add_argument("--allow-insecure-localhost")
            chromeOptions.add_argument("--allow-running-insecure-content")
            chromeOptions.accept_untrusted_certs = True
            chromeOptions.assume_untrusted_cert_issuer = True
            service_args = ["hide_console"]
            currentPath = (os.path.dirname(os.path.abspath(__file__)) +
                           "\\ChromeDriver\\" + chrome_browser_version +
                           "\\chromedriver.exe")

            while True:
                try:
                    print("\nCalling Driver")

                    # Creating an instance of chrome
                    driver = Chrome(
                        executable_path=currentPath,
                        options=chromeOptions,
                        service_args=service_args,
                    )
                    print("Driver Called")
                    driver.set_page_load_timeout(10)
                    driver.delete_all_cookies()

                    # open a page
                    driver.get("Enter Checking Site Here")
                    print("Getting Site")
                    try:
                        """
                        
                            remove this try except if the your wifi doesn't block websites
                        
                        """

                        # xpath if the website is blocked
                        element = driver.find_element_by_xpath(
                            "Enter xpath to an element in the blocked page")
                        print("Site Blocked\n")

                    except:
                        try:
                            # xpath to any thing in the website to make sure you are connected to the internet
                            element = driver.find_element_by_xpath(
                                "/Enter xpath to an element in the page")
                            print("Site Opening\n")

                        except:
                            try:
                                """
                                
                                    if your portal doesn't have auto redirect, uncomment the following line and type in your login url
                                
                                """

                                # driver.get("Paste Login Webiste URL Here")

                                # change the ids to those in your login website
                                # you can use developer mode to find the id of fields (use ctrl + shift + i)
                                # change the username and password to the required one
                                print("Trying To Login")
                                # select usnername field
                                element = driver.find_element_by_id(
                                    "Ending id of user input field")
                                print("User Found")
                                element.send_keys("Enter username")
                                print("User Inputted")
                                # select password field
                                element = driver.find_element_by_id(
                                    "Ending id of password input field")
                                print("Passwprd Found")
                                element.send_keys("Enter password")
                                print("Password Inputted")
                                # select submit button
                                element = driver.find_element_by_id(
                                    "Enter id of submit button")
                                print("Button Found")
                                element.click()
                                print("Logged In\n")
                            # except NoSuchElementException as ex:
                            #     print("Can't Login")
                            #     event.wait(120)
                            except Exception as ex:
                                print(
                                    "Can't login:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                                    .format(type(ex).__name__, ex.args))
                                event.wait(60)
                                continue

                except Exception as ex:
                    print(
                        "Error in loop:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                        .format(type(ex).__name__, ex.args))
                    try:
                        driver.quit()
                    except Exception as ex:
                        print(
                            "Error in Quitting:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                            .format(type(ex).__name__, ex.args))

                    event.wait(60)
                    continue

                try:
                    driver.quit()
                except Exception as ex:
                    print(
                        "Error in Quitting in loop:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                        .format(type(ex).__name__, ex.args))
                event.wait(300)
                continue

        except Exception as ex:
            print(
                "Error outside loop:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                .format(type(ex).__name__, ex.args))

        finally:
            try:
                driver.quit()
            except Exception as ex:
                print(
                    "Error in Quitting in final:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
                    .format(type(ex).__name__, ex.args))
            finally:
                print("Login Thread Exited")
Пример #8
0
def login(email):
    global valid
    global invalid
    global count
    colors = {
        'merah': Fore.RED,
        'hijau': Fore.GREEN,
        'putih': Fore.WHITE,
        'kuning': Fore.YELLOW,
        'biru': Fore.CYAN,
        'black': Fore.BLACK
    }
    if os.system == "nt":
        path = "chromedriver.exe"
    else:
        path = "chromedriver"
    url = "https://www.remitly.com/us/en/users/register"
    f = open("proxies.txt", "r")
    proxies = f
    for proxi in proxies:
        myproxy = proxi

    prox = Proxy()
    prox.proxy_type = ProxyType.MANUAL
    prox.http_proxy = myproxy
    prox.socks5_proxy = myproxy
    prox.ssl_proxy = myproxy
    capabilities = webdriver.DesiredCapabilities.CHROME
    prox.add_to_capabilities(capabilities)
    options = Options()
    options.headless = True
    options.add_argument('log-level=3')
    options.accept_untrusted_certs = True
    options.assume_untrusted_cert_issuer = True
    options.add_experimental_option('excludeSwitches', ['enable-logging'])
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-impl-side-painting")
    options.add_argument('--disable-gpu')
    options.add_argument("--disable-setuid-sandbox")
    options.add_argument("--disable-seccomp-filter-sandbox")
    options.add_argument("--disable-breakpad")
    options.add_argument("--disable-client-side-phishing-detection")
    options.add_argument("--disable-cast")
    options.add_argument("--disable-cast-streaming-hw-encoding")
    options.add_argument("--disable-cloud-import")
    options.add_argument("--disable-popup-blocking")
    options.add_argument("--ignore-certificate-errors")
    options.add_argument("--disable-session-crashed-bubble")
    options.add_argument("--disable-ipv6")
    options.add_argument("--allow-http-screen-capture")
    options.add_argument("--start-maximized")
    driver = webdriver.Chrome(path,
                              chrome_options=options,
                              desired_capabilities=capabilities)
    driver.delete_all_cookies()
    driver.get(url)
    driver.find_element_by_name('email_address').send_keys(email)
    driver.find_element_by_name('password').send_keys('jakarta321')
    Select(
        driver.find_element_by_xpath(
            '/html/body/div[2]/div[2]/div/div/form/div[3]/div/select')
    ).select_by_value('4')
    button = driver.find_element_by_xpath(
        '/html/body/div[2]/div[2]/div/div/form/div[4]/button')
    button.click()
    # response = driver.find_elements_by_xpath("/html/body/div[2]/div[2]/div/div/script[1]")
    sleep(1)
    response = driver.execute_script(
        "return document.getElementsByTagName('html')[0].innerHTML")
    if 'This email address is associated with another account.' in (
            str(response)):
        with myLock:
            Pac = email
            loglive.write(Pac)
            count = count + 1
            valid = valid + 1
            print("LIVE => " + email)
    else:
        with myLock:
            logdie.write(email)
            count = count + 1
            invalid = invalid + 1
            print("DIE => " + email)
Пример #9
0
def initialize_driver(headless=True,
                      with_ublock=False,
                      chromedriver_path=None):
    if not chromedriver_path:
        try:
            chromedriver_path = chromedriver_autoinstaller.install()
        except Exception as exception:
            log.critical(
                f"Failed to install the built-in chromedriver: {exception}\n"
                "download the correct version for your system at "
                "https://chromedriver.chromium.org/downloads and use the"
                "--chromedriver argument to point to the chromedriver executable"
            )
            sys.exit()

    log.info(f"Initialising chromedriver at {chromedriver_path}...")
    chrome_options = Options()

    chrome_options.add_argument(
        "--disable-blink-features=AutomationControlled")
    chrome_options.add_experimental_option(
        "excludeSwitches", ["enable-automation", "enable-logging"])
    chrome_options.add_experimental_option('useAutomationExtension', False)
    chrome_options.add_argument('--disable-dev-shm-usage')
    chrome_options.accept_untrusted_certs = True
    chrome_options.assume_untrusted_cert_issuer = True
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--allow-http-screen-capture")
    chrome_options.add_argument("--disable-impl-side-painting")
    chrome_options.add_argument("--disable-setuid-sandbox")
    chrome_options.add_argument("--disable-seccomp-filter-sandbox")

    if headless:
        chrome_options.add_argument("--headless")
        chrome_options.add_argument("window-size=1920,1080")
    chrome_options.add_argument("--log-level=3")
    chrome_options.add_argument("--silent")
    chrome_options.add_argument("--disable-logging")
    # allows selenium to accept cookies with a non-int64 'expiry' value
    chrome_options.add_experimental_option("w3c", False)
    # removes the 'DevTools listening' log message
    chrome_options.add_experimental_option("excludeSwitches",
                                           ["enable-logging"])

    if with_ublock:
        chrome_options.add_extension(
            os.path.join(os.getcwd(), "bin", "ublock", "ublock-extension.crx"))

    logs_path = os.path.join(os.getcwd(), "logs")
    if not (os.path.isdir(logs_path)):
        os.makedirs(logs_path)

    driver = webdriver.Chrome(
        executable_path=chromedriver_path,
        service_log_path=os.path.join(logs_path, "webdrive.log"),
        # Don't verify self-signed cert, should help with 502 errors
        # (https://github.com/wkeeling/selenium-wire/issues/55)
        # seleniumwire_options={"verify_ssl": False},
        options=chrome_options,
    )

    if with_ublock:
        log.debug("Configuring uBlock")

        # set up uBlock
        driver.get(
            "chrome-extension://ilchdfhfciidacichehpmmjclkbfaecg/settings.html"
        )

        # Un-hide the file upload button so we can use it
        element = driver.find_elements_by_class_name("hidden")
        driver.execute_script(
            "document.getElementsByClassName('hidden')[0].className = ''",
            element)
        # scroll down (for debugging)
        driver.execute_script("window.scrollTo(0, 2000)")
        uBlock_settings_file = str(
            os.path.join(os.getcwd(), "bin", "ublock", "ublock-settings.txt"))
        driver.find_element_by_id("restoreFilePicker").send_keys(
            uBlock_settings_file)
        try:
            WebDriverWait(driver, 3).until(EC.alert_is_present())
            # click ok on pop up to accept overwrite
            driver.switch_to.alert.accept()
        except TimeoutException as ex:
            log.error("Timeout waiting for ublock config overwrite alert")
        # leave uBlock config
        driver.get("about:blank")

    return driver