def stat_scraper(link, driver): options = Options() # options.headless = True options.add_extensions = '/Users/Philip/Documents/NBA prediction script/Incremental Pipelines/3.34.0_0' browser = webdriver.Chrome(executable_path=driver, chrome_options=options) browser.get(link) while True: try: wait = WebDriverWait(browser, 10).until( EC.visibility_of_element_located(( By.XPATH, '/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[2]/div[1]' ))) break except TimeoutException or NoSuchElementException: browser.refresh() print('failed to find page') logging.info('Failed to connect to page') browser.find_element_by_xpath( '/html/body/main/div[2]/div/div[2]/div/div/div[1]/div[1]/div/div/label/select/option[1]' ).click() browser.find_element_by_xpath( '/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[3]/div/div/select/option[1]' ).click() table = browser.find_element_by_class_name('nba-stat-table') content = table.get_attribute('innerHTML') df = pd.read_html(content)[0] browser.quit() return format_matchup(df)
def gen_browser(driver): options = Options() options.headless = True options.add_extensions = '/Users/Philip/Documents/NBA prediction script/Incremental Pipelines/3.34.0_0' return webdriver.Chrome(executable_path=driver, chrome_options=options)