def get_browser(url, proxy=""): from selenium import webdriver from selenium.webdriver.common.proxy import Proxy, ProxyType if proxy == "": browser = webdriver.Chrome( ) #replace with .Firefox(), or with the browser of your choice else: prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.https_proxy = proxy #prox.https_proxy = proxy #prox.socks_proxy = "ip_addr:port" #prox.ssl_proxy = "ip_addr:port" capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) browser = webdriver.Chrome(desired_capabilities=capabilities) browser.get(url) #navigate to the page #browser.close() return browser
requesting the driver to refresh the page after 1-minute sleep...') time.sleep(60) canvas = _wait_for_page_to_present(driver) continue if __name__ == '__main__': logging_.initialize(module_name='game_abstract_crawler', service_name='crawler') options = Options() options.headless = True options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') options.add_argument('--window-size=800,600') proxy = Proxy() proxy.http_proxy = 'localhost:8080' proxy.https_proxy = 'localhost:8080' capabilities = DesiredCapabilities.CHROME proxy.add_to_capabilities(capabilities) with Chrome(options=options, desired_capabilities=capabilities) as driver: try: main(driver) except Exception as e: _get_screenshot(driver, '99-エラー.png') logging.exception('Abort with an unhandled exception.') raise
"https://discordapp.com/api/webhooks/697993898659676230/09E2-tTgtVWc6NLuJ5NgaXyioE98A_VADBR3MbULkupaz-C_1UDX350L2VEDV46ams07" ) embed = Embed(description=product_url, color=0x5CDBF0, timestamp="now") embed.add_field(name="STOCK AVAILABILITY", value=stock) hook.send(embed=embed) while True: options = webdriver.ChromeOptions() options.add_argument('headless') desired_caps = options.to_capabilities() prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = "http://scraperapi.country_code=us:[email protected]:8001" prox.https_proxy = "https://scraperapi.country_code=us:[email protected]:8001" prox.add_to_capabilities(desired_caps) driver = webdriver.Chrome(desired_capabilities=desired_caps) url = 'https://www.amazon.com' driver.get(url) try: address_xpath = '//*[@id="nav-main"]/div[1]/div[2]/div/div[3]/span[2]/span/input' address = driver.find_element_by_xpath(address_xpath) except: address = None if address: print('checking address...') driver.find_element_by_xpath(address_xpath).click()