def __init__(self, driver_name, driver_executable_path, driver_arguments, browser_executable_path): webdriver_base_path = f'selenium.webdriver.{driver_name}' driver_klass_module = import_module(f'{webdriver_base_path}.webdriver') driver_klass = getattr(driver_klass_module, 'WebDriver') driver_options_module = import_module(f'{webdriver_base_path}.options') driver_options_klass = getattr(driver_options_module, 'Options') driver_options = driver_options_klass() if browser_executable_path: driver_options.binary_location = browser_executable_path for argument in driver_arguments: driver_options.add_argument(argument) driver_kwargs = { 'executable_path': driver_executable_path, f'{driver_name}_options': driver_options } driver_options.add_experimental_option( "excludeSwitches", ["enable-automation"]) driver_options.add_experimental_option('useAutomationExtension', False) self.driver = driver_klass(**driver_kwargs) stealth( self.driver, vendor='Google Inc.', platform='Win32', webgl_vendor='Intel Inc.', renderer="Intel Iris OpenGL Engine", fix_hairline=True, )
def session_create(config): log.info("Creating session") options = webdriver.ChromeOptions() if config.get('headless', False) is True: log.info("Headless mode") options.add_argument("--headless") if os.path.isfile("./chrome-win/chrome.exe"): log.info("Found ./chrome-win/chrome.exe") options.binary_location = "./chrome-win/chrome.exe" driver = webdriver.Chrome(options=options) stealth( driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) log.info("New session is: %s %s" % (driver.session_id, driver.command_executor._url)) return driver
def initChromeDriver(): chrome_options = webdriver.ChromeOptions() # prefs = {"download.default_directory" : "/some/path"} # chromeOptions.add_experimental_option("prefs",prefs) chrome_options.add_argument('--headless') # chrome_options.add_argument('--no-sandbox') # chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.add_argument('--incognito') chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"]) chrome_options.add_experimental_option('useAutomationExtension', False) driver = webdriver.Chrome(os.environ.get('chromedriver'), chrome_options=chrome_options) # driver = webdriver.Chrome('./chromedriver',chrome_options=chrome_options) stealth( driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) return driver
def initChromeDriver(): options = webdriver.ChromeOptions() # options.add_argument('--headless') #-- Uncomment the below two lines while running on MAC/LINUX --# # options.add_argument('--no-sandbox') # options.add_argument('--disable-dev-shm-usage') options.add_argument('--incognito') options.add_argument("--start-maximized") options.add_argument('ignore-certificate-errors') options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) prefs = {"download.default_directory": f"{os.getcwd()}\\downloads\\"} options.add_experimental_option("prefs", prefs) #driver = webdriver.Chrome(os.environ.get('chromedriver'), options=options) driver = webdriver.Chrome('./chromedriver', options=options) stealth( driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) return driver
def browser_data(): options = webdriver.ChromeOptions() options.add_argument("start-maximized") options.add_argument("--headless") # option.add_argument("--no-sandbox") # options.add_argument("--disable-gpu") # options.addArguments("--disable-dev-shm-usage") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) driver = webdriver.Chrome(options=options) stealth( driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) path = str(os.getcwd()).replace('\\', '/') + "/tests/static/test.html" url = "https://bot.sannysoft.com/" if os.name == 'nt': url = 'file:///' + path else: url = 'file://' + path print(url) driver.get(url) time.sleep(10) metrics = driver.execute_cdp_cmd('Page.getLayoutMetrics', {}) width = math.ceil(metrics['contentSize']['width']) height = math.ceil(metrics['contentSize']['height']) screenOrientation = dict(angle=0, type='portraitPrimary') driver.execute_cdp_cmd( 'Emulation.setDeviceMetricsOverride', { 'mobile': False, 'width': width, 'height': height, 'deviceScaleFactor': 1, 'screenOrientation': screenOrientation, }) clip = dict(x=0, y=0, width=width, height=height, scale=1) opt = {'format': 'png'} if clip: opt['clip'] = clip result = driver.execute_cdp_cmd('Page.captureScreenshot', opt) html = driver.page_source driver.quit() return html, result
def __init__(self, showWindow=True, proxy=None, downloadPath=None): options = webdriver.ChromeOptions() options.add_argument("--disable-dev-shm-usage") options.add_argument("--no-sandbox") options.add_argument('--disable-web-security') options.add_argument('--allow-running-insecure-content') options.add_argument("user-agent=DN") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) if downloadPath is not None and isinstance(downloadPath, str): absolute_path = os.path.abspath(downloadPath) if not os.path.isdir(absolute_path): raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), absolute_path) options.add_experimental_option( 'prefs', {'download.default_directory': absolute_path}) if proxy is not None and isinstance(proxy, str): options.add_argument("--proxy-server={}".format(proxy)) if not showWindow: options.headless = True self.driver = webdriver.Chrome(options=options) self.Key = Keys self.errors = [] stealth( self.driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) for function in [ 'add_cookie', 'delete_all_cookies', 'delete_cookie', 'execute_script', 'execute_async_script', 'fullscreen_window', 'get_cookie', 'get_cookies', 'get_log', 'get_network_conditions', 'get_screenshot_as_base64', 'get_screenshot_as_file', 'get_screenshot_as_png', 'get_window_position', 'get_window_rect', 'get_window_size', 'maximize_window', 'minimize_window', 'implicitly_wait', 'quit', 'refresh', 'save_screenshot', 'set_network_conditions', 'set_page_load_timeout', 'set_script_timeout', 'set_window_position', 'set_window_rect', 'start_client', 'start_session', 'stop_client', 'switch_to_alert' ]: setattr(self, function, getattr(self.driver, function))
def setup_browser(self): stealth( self.browser, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) self.get_params(self.browser) self.browser.execute_script(get_acrawler())
def setup_browser(self): stealth( self.browser, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) self.get_params(self.browser) # NOTE: Slower than playwright at loading this because playwright can ignore unneeded files. self.browser.get("https://www.tiktok.com/@redbull") self.browser.execute_script(get_acrawler()) self.browser.execute_script(get_tt_params_script())
def __init__(self): options = webdriver.ChromeOptions() options.add_argument("start-maximized") #options.add_argument("--headless") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) self.driver = webdriver.Chrome(options=options) stealth( self.driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, )
def get_oauth2_token_stack(client): try: br = webdriver.Firefox(FirefoxDriverManager().install()) except Exception as e: try: options = webdriver.ChromeOptions() options.add_argument("--headless") options.add_argument("user-agent=DN") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) br = webdriver.Chrome(ChromeDriverManager().install(), options=options) stealth( br, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) except Exception as e: return None #br.get('https://accounts.google.com/signin/oauth/identifier?client_id=717762328687-iludtf96g1hinl76e4lc1b9a82g457nn.apps.googleusercontent.com&as=JS6BM8cjL-8j9votansdkw&destination=https%3A%2F%2Fstackauth.com&approval_state=!ChRoYWVvLUlNMk5hSXJWUGlaSVl2WBIfc3lSa0lueENpb29lSU5vbEVpbVNxcUZGaGNkSEJoYw%E2%88%99AJDr988AAAAAXlBKc7PzEomxSzgNqd4wLptVlf0Ny3Qx&oauthgdpr=1&xsrfsig=ChkAeAh8T8JNDxCf2Zah5fb_rQ55OMiF8KmMEg5hcHByb3ZhbF9zdGF0ZRILZGVzdGluYXRpb24SBXNvYWN1Eg9vYXV0aHJpc2t5c2NvcGU&flowName=GeneralOAuthFlow') br.get('https://stackoverflow.com/') time.sleep(2) br.find_element_by_xpath('/html/body/header/div/ol[2]/li[2]/a[1]').click() time.sleep(0.5) br.find_element_by_xpath( '/html/body/div[3]/div[2]/div/div[2]/button[1]').click() time.sleep(0.5) br.find_element_by_id('Email').send_keys(client.userId) #print(Fore.YELLOW + ' SENT EMAIL LOGIN ') time.sleep(0.5) br.find_element_by_xpath( '/html/body/div/div[2]/div[2]/div[1]/form/div/div/input').click() time.sleep(0.5) br.find_element_by_id('password').send_keys(client.password) #print(Fore.YELLOW + time_format() + ' SENT PASSWORD LOGIN ') br.find_element_by_id('submit').click() #print(Fore.BLUE + ' SUCCESSFULLY LOGGED IN ') logged = True
def initChromeDriver(): options = webdriver.ChromeOptions() options.add_argument("--log-level=3") options.add_argument('--start-maximized') options.add_argument('--headless') options.add_argument('--incognito') options.add_argument('ignore-certificate-errors') options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) driver = webdriver.Chrome(ChromeDriverManager().install(), options=options) stealth( driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) return driver
def __init__(self, coords=None): options = webdriver.ChromeOptions() options.add_argument("start-maximized") options.add_argument("start-maximized") options.add_argument("--disable-extensions") # options.add_argument("--headless") options.add_experimental_option( "prefs", { "profile.default_content_setting_values.geolocation": 1, "profile.password_manager_enabled": False, "credentials_enable_service": False }, ) options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option("useAutomationExtension", False) self.driver = webdriver.Chrome(options=options) # Madison, WI Coords #coords = { # "latitude": 43.073051, # "longitude": -89.40123, # "accuracy": 98, #} if coords != None: self.driver.execute_cdp_cmd("Page.setGeolocationOverride", coords) stealth( self.driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, )
def initChromeDriver(): chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--headless') #-- Uncomment the below two lines while running on MAC/LINUX --# # chrome_options.add_argument('--no-sandbox') # chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.add_argument('--incognito') chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"]) chrome_options.add_experimental_option('useAutomationExtension', False) # driver = webdriver.Chrome(os.environ.get('chromedriver'), chrome_options=chrome_options) driver = webdriver.Chrome('./chromedriver', chrome_options=chrome_options) stealth( driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) return driver
USERNAME = input("User Name : ") PASSWORD = input("Password : "******"start-maximized") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) options.add_experimental_option('excludeSwitches', ['enable-logging']) # stealth 기능 stealth( driver, user_agent='DN', languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) # Before Login, using stealth login(USERNAME, PASSWORD) stealth( driver, user_agent= 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36', languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.",
import sys from seleniumwire import webdriver from selenium.webdriver.common.keys import Keys from selenium_stealth import stealth name = sys.argv[1] email = sys.argv[2] password = sys.argv[3] options = webdriver.ChromeOptions() options.add_argument("--disable-blink-features=AutomationControlled") driver = webdriver.Chrome(options=options) stealth( driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) driver.get( "http://demo-rails-app-testing.ipm-corporation.com/signup?disallow_selenium=true" ) element = driver.find_element_by_id("user_name") element.send_keys(name) element = driver.find_element_by_id("user_email") element.send_keys(email) element = driver.find_element_by_id("user_password") element.send_keys(password) element = driver.find_element_by_id("user_password_confirmation") element.send_keys(password)
def browser(proxy): global workers # email = accountinfo.get_email() # if not email: # workers -= 1 # return options = uc.ChromeOptions() ua = UserAgent() userAgent = ua.chrome options.add_argument(f'user-agent={userAgent}') #options.add_argument("--headless") options.add_argument("--proxy-server={}://{}".format(proxyType, proxy)) options.add_argument("--disable-blink-features=AutomationControlled") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) options.add_argument("--enable-features=ReaderMode") options.add_argument('--ignore-ssl-errors=yes') options.add_argument('--ignore-certificate-errors') prefs1 = { "profile.managed_default_content_settings.images": 1, "profile.managed_default_content_settings.popups": 2, "profile.default_content_setting_values.notifications": 2, "profile.managed_default_content_settings.geolocation": 2, "profile.managed_default_content_settings.stylesheets": 2, } options.add_experimental_option("prefs", prefs1) options.binary_location = r"C:\Program Files (x86)\BraveSoftware\Brave-Browser\Application\brave.exe" try: driver = uc.Chrome(chrome_options=options) stealth( driver, languages=["en-US", "en"], user_agent=userAgent, vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) url = "https://discord.com/register" #url = "https://accounts.google.com/signin/v2/identifier?service=youtube&uilel=3&passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Faction_handle_signin%3Dtrue%26app%3Ddesktop%26hl%3Den%26next%3Dhttps%253A%252F%252Fwww.youtube.com%252F&hl=en&ec=65620&flowName=GlifWebSignIn&flowEntry=ServiceLogin" try: driver.get(url) except WebDriverException: workers -= 1 driver.quit() return Thread(target=await_tab, args=(driver, ), daemon=True).start() sleeptime = 0.4 email = "testing1234crazytown{}@gmail.com".format( random.randint(0, 100000)) username = "******".format(random.randint(0, 100000)) password = "******" selection_month = random.choice([ 'December', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November' ]) selection_day = random.randint(1, 28) try: wait = WebDriverWait(driver, 30) enter_searchbar = wait.until( EC.presence_of_element_located((By.NAME, "email"))) slow_typing(enter_searchbar, email, 20) time.sleep(sleeptime) enter_user = driver.find_element_by_name("username") slow_typing(enter_user, username, 20) time.sleep(sleeptime) enter_password = driver.find_element_by_name("password") slow_typing(enter_password, password, 20) time.sleep(sleeptime) enter_month = driver.find_element_by_id("react-select-2-input") slow_typing(enter_month, selection_month, 20) enter_month.send_keys(Keys.ENTER) time.sleep(sleeptime) enter_day = driver.find_element_by_id("react-select-3-input") slow_typing(enter_day, selection_day, 20) enter_day.send_keys(Keys.ENTER) time.sleep(sleeptime) enter_year = driver.find_element_by_id("react-select-4-input") slow_typing(enter_year, random.randint(1988, 1997), 20) enter_year.send_keys(Keys.ENTER) time.sleep(sleeptime) try: driver.find_element_by_xpath( "//input[@type='checkbox']").click() except Exception: pass driver.find_element_by_xpath("//button[@type='submit']").click() try: WebDriverWait(driver, 30).until( EC.frame_to_be_available_and_switch_to_it( (By.XPATH, '//iframe[contains(@title, "widget")]'))) WebDriverWait(driver, 10).until( EC.element_to_be_clickable( (By.CSS_SELECTOR, "div#checkbox.checkbox"))).click() print("Found one!") except Exception: print("failed to find iframe!") workers -= 1 driver.quit() return except Exception as e: print(e) except Exception as e: print(e) time.sleep(30) workers -= 1 driver.quit() return
def parse(self, response): driver = response.meta['driver'] stealth( driver, languages=["en-US", "en"], vendor="Google Inc.", platform="Win32", webgl_vendor="Intel Inc.", renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) driver.maximize_window() time.sleep(3) for _, val in self.df.iterrows(): if val['Id'] not in self.idList: self.idList.append(val['Id']) print(val['gSearchQuery']) #input() driver.get( f"https://www.google.com/search?q={val['gSearchQuery']}") #input() time.sleep(1) WebDriverWait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, "//div[@class='g']"))) #try: # self.cntr += 1 # if self.cntr % 100 == 0: # driver.refresh() # time.sleep(random.randint(5,10)) # try: # WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//div[@class='g']"))) # except: # input() # SEARCHING FOR THE QUERY STRING # # inputElem = driver.find_element_by_xpath("//input[@title='Search']") # inputElem.clear() # inputElem.send_keys(val['gSearchQuery']) # serachBtnElem = driver.find_element_by_xpath("//button[@type='submit']") # driver.execute_script("arguments[0].click()", serachBtnElem) # time.sleep(1.5) #WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.XPATH, "//div[@class='g']"))) # GETTING THE PAGE HEIGHT & SCROLLING GRADUALLY THROUGH THE PAGE TO LAOD ALL AVAILABLE LISTINGS # # height = driver.execute_script("return document.body.scrollHeight") # for i in range(1, (height//300)+1): # driver.execute_script(f"window.scrollTo(0, {i*300});") # time.sleep(0.1) # driver.execute_script(f"window.scrollTo(0, -5000);") html = driver.page_source respObj = Selector(text=html) website = respObj.xpath( "//div[text()='Website']/parent::a/@href").get() phone = respObj.xpath( "normalize-space(//span[contains(@aria-label, 'phone number')]/text())" ).get() if not website: website = respObj.xpath( "//div[text()='Website']/parent::div/parent::a/@href" ).get() if not phone: phone = respObj.xpath( "normalize-space(//span[contains(@aria-label, 'Phone Number')]/text())" ).get() if not phone: phone = respObj.xpath( "normalize-space((//div[text()='Website']/parent::div/parent::a/preceding-sibling::a)[1]//span[contains(@class,'details')]/div[2]/span[last()]/text())" ).get() if not phone: phone = respObj.xpath( "normalize-space((//div[text()='Website']/parent::div/parent::a/preceding-sibling::a)[1]//span[contains(@class,'details')]/div[3]/span[last()]/text())" ).get() if not phone: phone = respObj.xpath( "normalize-space(//div[text()='A']/parent::div/span/div[2]/span[last()]/text())" ).get() yield { 'Id': val['Id'], 'City': val['State'], 'Area': val['City'], 'Restaurant Name': val['Restaurant name'], 'Average Rating': val['Average rating'], 'Number Of Reviews': val['Number of reviews'], 'Search Queries': val['gSearchQuery'], 'Website': self.extractUrl(website), 'Phone': phone }