def _authoriz_upd(self): url = "https://app.zakupki360.ru/login" chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--headless') with webdriver.Chrome(executable_path="./chromedriver", options=chrome_options) as driver: driver.implicitly_wait(10) driver.get(url) driver.find_element_by_id("mat-input-0").send_keys( config.authoriz['login']) driver.find_element_by_id("mat-input-1").send_keys( config.authoriz['password']) driver.find_element_by_tag_name('button').click() driver.find_element_by_class_name("items") resp = tuple({ req.headers['Authorization'] for req in driver.requests if req.response }) try: key = resp[0] if resp[0] else resp[1] with open("auth_key", "w") as f_out: f_out.write(key) return key except Exception as e: print(resp) print(e)
def __init__(self, *args, **kwargs): chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--headless') chrome_options.add_argument('connection_keep_alive=False') chrome_options.add_argument( 'disable-blink-features=AutomationControlled') chrome_options.add_argument( "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:79.0) Gecko/20100101 Firefox/79.0" ) options = { 'backend': 'mitmproxy', 'disable_encoding': True # Tell the server not to compress the response # Set the backend to mitmproxy } self.driver = webdriver.Chrome('chromedriver', options=chrome_options, seleniumwire_options=options) self.driver.maximize_window() self.driver.execute_cdp_cmd( "Page.addScriptToEvaluateOnNewDocument", { "source": """ Object.defineProperty(navigator, 'webdriver', { get: () => undefined }) """ })
def scrape_kite_cookies(clientid, passwd, pin, headless=True): base_url = 'https://kite.zerodha.com/' demo_chart_url = 'https://kite.zerodha.com/chart/web/tvc/INDICES/NIFTY%2050/256265' options = webdriver.ChromeOptions() if headless: options.add_argument('headless') # For headless mode # endif driver = webdriver.Chrome(chrome_options=options) driver.get(base_url) # Login using clientid and password base_inps = driver.find_elements_by_tag_name('input') base_inps[0].send_keys(clientid) base_inps[1].send_keys(passwd) base_button = driver.find_element_by_tag_name('button') base_button.click() time.sleep(2) # Specify PIN der_inp = driver.find_element_by_tag_name('input') der_inp.send_keys(pin) der_button = driver.find_element_by_tag_name('button') der_button.click() # Load a demo chart and sleep driver.get(demo_chart_url) time.sleep(2) return driver.get_cookies()
def init_driver(self): # settings selenium driver_options = webdriver.ChromeOptions() if self.browser_executable_path: driver_options.binary_location = self.browser_executable_path # # switch user_agent, proxy # if hasattr(self, 'agent'): # driver_options.add_argument(f"--user-agent='{next(self.agent)}'") # if hasattr(self, 'proxy'): # driver_options.add_argument(f"--proxy-server='{next(self.proxy)}'") # add SELENIUM_DRIVER_ARGUMENTS options for argument in self.driver_arguments: driver_options.add_argument(argument) # add experimental option: on/off image and notify chrome_prefs = {"disk-cache-size": 4096} if not self.driver_load_img: chrome_prefs.update( { "profile.default_content_settings": {"images": 2}, "profile.managed_default_content_settings": {"images": 2} } ) if self.driver_disable_notify: chrome_prefs.update( {"profile.default_content_setting_values.notifications": 2} ) driver_options.add_experimental_option("prefs", chrome_prefs) self.driver = webdriver.Chrome( executable_path=self.driver_executable_path, options=driver_options )
def __init__(self, location, page): if location == "": location = r"C:\Users\clone\Desktop" options = webdriver.ChromeOptions() options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_argument('--disable-extensions') options.add_argument("--incognito") options.add_argument("--disable-plugins-discovery") options.add_argument("--headless") options.add_argument("--log-level=OFF") self.parser = webdriver.Chrome( executable_path=r"C:\Users\clone\Desktop\chromedriver.exe", chrome_options=options) self.page = self.parser.get(page) name = "" for letter in self.fileName(): if letter != ":" and letter != "\\": name += letter self.location = os.path.join(location, name + ".mp4") #Creates file name & location time.sleep(5) site = self.siteGetter() header = self.headerGetter() self.parser.quit() self.properRipper(site, header, page, self.location)
def __init__(self, firstName, lastName, email, password): self.firstName = firstName self.lastName = lastName self.email = email self.password = password #create a new webdriver with a header options = webdriver.ChromeOptions() options.add_experimental_option('excludeSwitches', ['enable-logging']) options.add_argument('--ignore-certificate-errors') options.add_argument('--incognito') #remove headless option if you want to see the program work live #options.add_argument('--headless') options.add_argument( 'user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36"' ) self.driver = webdriver.Chrome(ChromeDriverManager().install(), options=options) #used to manage header requests self.driver.request_interceptor = interceptor curURL = self.driver.current_url self.driver.get("http://localhost:3000") wait(self.driver, 15).until(EC.url_changes(curURL)) self.driver.maximize_window() self.register() self.login()
def get_info(link): reviews = [] options = 1 driver = 1 try: options = webdriver.ChromeOptions() options.add_argument('--ignore-certificate-errors') options.add_argument('--incognito') options.add_argument('--headless') driver = webdriver.Chrome(executable_path=binary_path, chrome_options=options) driver.get(link) wait = WebDriverWait(driver,120) wait.until(wait_requests) except TimeoutException: print("Failed to load search bar at www.google.com") except Exception as e: print(e) finally: for request in driver.requests: if request.response and request.response.headers['Content-Type'] == 'application/json' and request.path == 'https://www.tripadvisor.es/data/graphql/batched': response = json.loads(request.response.body) for r in response: if 'locations' in r['data']: try: for location in r['data']['locations']: if 'reviewListPage' in location: for review in location['reviewListPage']['reviews']: r = Review(review['id'], review['userId'], review['title'], review['text'], review['language']) reviews.append(r) except Exception as e: "" driver.quit() return reviews
def get_chrome_driver(driver_path): """ Instantiates a Chrome webdriver object Args: driver_path: path to the Chrome driver Returns: webdriver: returns a webdriver object """ options = webdriver.ChromeOptions() options.page_load_strategy = 'normal' options.add_argument("--disable-logging") options.add_argument("--disable-login-animations") options.add_argument("--disable-notifications") options.add_argument("--disable-default-apps") options.add_argument("--disable-extensions") options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) options.add_argument("--disable-dev-shm-usage") options.add_argument("--disable-browser-side-navigation") options.add_argument("--headless") options.add_argument("--hide-scrollbars") options.add_argument('--log-level 3') options.add_argument("--incognito") options.add_argument("--no-zygote") options.add_argument('--disable-gpu') options.add_argument('--no-sandbox') options.add_argument('--disable-infobars') driver = webdriver.Chrome(driver_path, options=options) driver.set_page_load_timeout(PAGE_LOAD_TIMEOUT_SEC) return driver
def _start_chrome_driver(self) -> webdriver: """Using Selenium start the google chrome browser headless. All the browser requests and responses(har_fit data) will be recorded using a BrowsermobProxy proxy server. :return: Google chrome driver object. """ chrome_options = webdriver.ChromeOptions() prefs = {"profile.default_content_setting_values.notifications": 2} chrome_options.add_experimental_option("prefs", prefs) chrome_options.set_capability( 'proxy', { 'httpProxy': f'{self.proxy.proxy}', 'noProxy': '', 'proxyType': 'manual', 'sslProxy': f'{self.proxy.proxy}' }) # chrome_options.add_argument("--headless") driver = webdriver.Chrome(chrome_options=chrome_options) driver.set_page_load_timeout(999) driver.delete_all_cookies() return driver
def __init__(self, email, username, password, proxy=None): options = webdriver.ChromeOptions() options.add_experimental_option("excludeSwitches", ["enable-logging"]) self.seleniumwire_options = {'proxy': { 'http': proxy, 'https': proxy } } """if proxy: options.add_argument('--proxy-server=%s' % proxy)""" self.platform = platform.system() #Support for linux system """if self.platform == 'Windows': self.driver = webdriver.Chrome(options=options, executable_path=r"chromedriver.exe") else: self.driver = webdriver.Chrome(options=options, executable_path=r"chromedriver")""" #Webdrivermanager install chromedriver binary as per system's platform if proxy: #check if proxy is working and also update timezone as per proxy to avoid detection try: self.timezone = requests.get('http://ip-api.com/json',proxies = self.seleniumwire_options['proxy'],timeout = 10).json()['timezone'] self.tz_params = {'timezoneId': self.timezone} self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install(),seleniumwire_options=self.seleniumwire_options) except: self.timezone = None free_print(f"{Fore.LIGHTMAGENTA_EX}[!]{Style.RESET_ALL} " + 'Skipping Proxy as Connection Issue Have you entered it in the correct format?') self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install()) else: self.timezone = None self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install()) self.email= email self.username = username self.password = password
def options(self): chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--headless') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-dev-shm-usage') print('chrome options loaded') return chrome_options
def build(cfg, fetch_driver=True): """ builds a selenium-webdriver object with the specified configuration :param cfg: Configuration object :param fetch_driver: bool (default=True) fetches driver binaries :return: selenium-wire Webdriver object """ if cfg.proxy is not None: options = cfg.proxy.create_options() else: options = {} if cfg.driver in WebDriver.FIREFOX_DRIVER_NAMES: d = webdriver.Firefox o = webdriver.FirefoxOptions() if cfg.profile is None: p = webdriver.FirefoxProfile() else: p = webdriver.FirefoxProfile(cfg.profile) p.set_preference("general.useragent.override", cfg.user_agent) p.set_preference("media.volume_scale", "0.0") ''' if cfg.proxy is not None: p = cfg.proxy.update_preferences(p) ''' elif cfg.driver in WebDriver.CHROME_DRIVER_NAMES: d = webdriver.Chrome o = webdriver.ChromeOptions() o.add_argument("user-agent={0}".format(cfg.user_agent)) if cfg.user_data_dir: o.add_argument("user-data-dir={0}".format(cfg.user_data_dir)) ''' if cfg.proxy is not None: o.add_argument("--proxy-server={0}".format(cfg.proxy.for_chrome())) ''' p = None else: raise NotImplementedError if fetch_driver: Loader.fetch(cfg.executable_path, cfg.debug, cfg.driver) o.binary_location = cfg.executable_path o.headless = cfg.headless if cfg.driver in WebDriver.FIREFOX_DRIVER_NAMES: if cfg.proxy is None: return d(p, cfg.binary, options=o) else: '''return d(p, cfg.binary, options=o, proxy=cfg.proxy, seleniumwire_options=options)''' return d(p, cfg.binary, options=o, seleniumwire_options=options) elif cfg.driver in WebDriver.CHROME_DRIVER_NAMES: if cfg.proxy is None: return d(options=o) else: '''return d(options=o, proxy=cfg.proxy, seleniumwire_options=options)''' return d(options=o, seleniumwire_options=options)
def __buildDriver__(self, driver_options): # Ubuntu if (os.name == 'posix'): chromedriver = 'chromedriver' # Windows if (os.name == 'nt'): chromedriver = 'chromedriver.exe' if chromedriver: if config.DRIVER_NAME == 'Chrome': logger.info("Using Chrome Driver ...") options = webdriver.ChromeOptions() options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) options.add_experimental_option('w3c', False) for driver_option in driver_options: # if "--proxy-server" in driver_option: # print(f"\nADDING PROXY: [{driver_option}]\n") options.add_argument(driver_option) capabilities = None if config.PROXY: # print(f"\nADDING PROXY: [{config.PROXY}]\n") # prox = Proxy() # prox.proxy_type = ProxyType.MANUAL # prox.http_proxy = config.PROXY # # prox.socks_proxy = config.PROXY # prox.ssl_proxy = config.PROXY capabilities = webdriver.DesiredCapabilities.CHROME capabilities['loggingPrefs'] = {'performance': 'ALL'} # prox.add_to_capabilities(capabilities) if capabilities: self.driver = webdriver.Chrome( desired_capabilities=capabilities, options=options) # self.driver = config.DRIVER_NAME(desired_capabilities=capabilities, options=options) else: self.driver = webdriver.Chrome(chromedriver, options=options) # self.driver = config.DRIVER_NAME(chromedriver, options=options) else: logger.info("Using Firefox Driver ...") self.driver = webdriver.Firefox() self.min_wait = WebDriverWait(self.driver, 5) self.max_wait = WebDriverWait(self.driver, 20) self.els_css = self.driver.find_elements_by_css_selector self.el_css = self.driver.find_element_by_css_selector self.els_xpath = self.driver.find_elements_by_xpath self.el_xpath = self.driver.find_element_by_xpath self.driver.set_script_timeout(30) return self.driver
def test_wait_for_request_headless_chrome(self): # https://github.com/wkeeling/selenium-wire/issues/6 chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--headless") def get_page_response(url): # check for current os if os.name == 'posix': # osx driver_path = '/usr/local/bin/chromedriver' elif os.name == 'nt': # win32 driver_path = 'C:\chromedriver\chromedriver' else: print('Unknown operating system!!!') exit() driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=driver_path) driver.get(url) request = driver.wait_for_request(url, timeout=3) print(request) self.assertEqual(request.path, 'https://www.google.com/') get_page_response('https://www.google.com')
def load_browser(): global driver global ublock_guid chrome_options = webdriver.ChromeOptions() # chrome_options.binary_location = "/applications/developer/google\ chrome.app/Contents/MacOS/Google\ Chrome" # chrome_options.add_argument("--disable-web-security") # messes up ublock # chrome_options.add_argument('--headless') chrome_options.add_extension('ubo_1_30_4_0.crx') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-gpu') chrome_options.add_argument("--ignore-certificate-errors") chrome_options.add_argument("--enable-javascript") chrome_options.add_argument("--disable-chrome-google-url-tracking-client") chrome_options.add_argument("--safebrowsing-disable-download-protection") chrome_options.add_argument("--disable-domain-reliability") chrome_options.add_argument("--allow-running-insecure-content") chrome_options.add_argument( "--unsafely-treat-insecure-origin-as-secure=http://host:port") chrome_options.add_argument( '--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36' ) driver = webdriver.Chrome(executable_path="./chromedriver", options=chrome_options) driver.header_overrides = {'Referer': 'com.google.android.gm'} # display = Display(visible=0, backend="xephyr", size=(800, 600)) # display.start() extension_uri = "chrome-extension://cjpalhdlnbpafiamejdnhcphjbkeiagm/logger-ui.html?popup=1#_" driver.implicitly_wait(page_delay) driver.get(extension_uri) ublock_guid = driver.current_window_handle
def setup_browser(self): # Start Headless Display try: self.display = Display(visible=False, size=(self.width, self.height)) self.display.start() except EasyProcessError as e: self.headless: bool = True # Setup Chrome/Chromium - https://peter.sh/experiments/chromium-command-line-switches/ self.options = webdriver.ChromeOptions() self.options.add_argument('--disable-blink-features=AutomationControlled') # Disable WebDriver Detection self.options.add_argument(f'--user-agent={self.user_agent}') # To Identify Bot To Server Owners # Disable Code That Might Take CPU Cycles self.options.add_argument('--disable-extensions') # Chrome Apps/Extensions # self.options.add_argument('--disable-plugins') # No Longer Exists (Used To Be For Flash/Java/etc...) self.options.add_argument('--disable-notifications') # Disable Website Notification Capabilities self.options.add_argument('--disable-sync') # Disable Syncing Chrome Account Data # Force Lite Mode Pages # self.options.add_argument('--force-enable-lite-pages') # Force Lite Mode Pages # To Disable Accidental Audio Playing self.options.add_argument('--mute-audio') # Disables Audio Playing # self.options.add_argument('--incognito') # Incognito Mode # self.options.add_argument('--no-referrers') # Hide Referral Websites # self.options.add_argument('--no-experiments') # Disable Chrome Flags # self.options.add_argument('--remote-debugging-port=9222') # For Debugging if self.headless: # print("No V-Screen Available!!! Starting Headless!!!") self.options.add_argument('--headless')
def browsSart(self): try: uio = [ 'https://www.junno.co/news', 'https://www.junno.co/famouspeople', 'https://www.junno.co/lyrics', 'https://www.junno.co/shows/3', 'https://www.junno.co/movies/1' ] ip = self.rad_c() chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--headless") chrome_options.binary_location = os.environ.get( "GOOGLE_CHROME_BIN") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument( 'window-size=1920,{m}'.format(m=random.randint(708, 1080))) chrome_options.add_argument("--no-sandbox") self.driver = webdriver.Chrome( executable_path=os.environ.get("CHROMEDRIVER_PATH"), chrome_options=chrome_options) ty = self.myip() print(ty) self.driver._client.set_header_overrides( headers={ 'X-Forwarded-For': '{n}'.format(n=ip), 'Via': '{n}'.format(n=ip), 'X-Real-IP': '{n}'.format(n=ip), 'True-Client-IP': '{n}'.format(n=ip), 'Forwarded': "'for={n}:{m}'".format(n=ip, m=random.randint(20000, 60000)), 'User-Agent': '{n}'.format(n=random.choice(self.n)) }) self.driver.get(uio[random.randint(1, len(uio) - 1)]) element_present = EC.presence_of_element_located( (By.XPATH, '//body')) WebDriverWait(self.driver, 5).until(element_present) self.driver.find_element_by_id('{n}'.format(n=ty)).click() element_present = EC.presence_of_element_located( (By.XPATH, '//body')) data = WebDriverWait(self.driver, 5).until(element_present) time.sleep(1) self.driver.delete_all_cookies() self.driver.execute_script('window.localStorage.clear();') self.driver.execute_script('window.sessionStorage.clear();') self.driver.quit() print('good') except Exception: print('bad error CheckProxy') if self.driver: self.driver.quit()
def __init__(self, settings): self.settings = settings self.fakemail = FakeMail() self.chrome_options = webdriver.ChromeOptions() self.chrome_options.add_experimental_option('excludeSwitches', ['enable-logging'])
def __init__(self, proxy_class, price_class): """Создаем экземпляр бота""" Thread.__init__(self) if settings.DEBUG: print('CREATE BOT') self.proxies = proxy_class self.price = price_class self.no_proxy = False self.use_proxy = 1 if self.use_proxy == 2: self.proxy = '' else: self.proxy, clean_proxy = self.proxies.get_new_proxy() """Подключаем настройки прокси к экземпляру бота""" options = {'proxy': self.proxy} wind_width = random.randint(1000, 1920) wind_height = random.randint(750, 1080) with open( f'{settings.MEDIA_DIR}/user-agents/user_agents_for_chrome_pk.txt', 'r', encoding='utf-8') as f: user_agent = random.choice([line.strip() for line in f]) self.options = webdriver.ChromeOptions() self.options.add_argument("ignore-certificate-errors") self.options.add_argument(f'window-size={wind_width},{wind_height}') self.options.add_argument("--disable-blink-features") self.options.add_argument( "--disable-blink-features=AutomationControlled") self.options.add_argument("--headless") self.options.add_argument('--disable-extensions') self.options.add_argument('--profile-directory=Default') """Добавялем юзер агент из списка""" self.options.add_argument(f"user-agent={user_agent}") """Создаем экземпляр браузера""" if self.use_proxy == 1: self.driver = webdriver.Chrome( options=self.options, executable_path=f'{settings.MEDIA_DIR}/drivers/chromedriver', seleniumwire_options=options, ) else: self.driver = webdriver.Chrome( options=self.options, executable_path=f'{settings.MEDIA_DIR}/drivers/chromedriver', ) """Создаем необходимые переменные для работы""" self.reboot = False self.search_status = False self.step = 0 self.target = []
def get_chrome(): chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('headless') chrome_options.add_argument("--mute-audio") chrome_options.add_experimental_option("excludeSwitches", ["enable-logging"]) return webdriver.Chrome(ChromeDriverManager().install(), chrome_options=chrome_options)
def launch_browser(headers=None, user_agent=None, proxy=None, browser_type="Firefox"): options = {} if proxy: proxy = { "http": proxy, "https": proxy, } options["proxy"] = proxy if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): directory = sys._MEIPASS else: directory = os.path.dirname(__file__) driver = None if browser_type == "Firefox": matches = ["geckodriver.exe", "geckodriver"] driver_paths = list( map(lambda match: os.path.join(directory, match), matches)) found_paths = [ driver_path for driver_path in driver_paths if os.path.exists(driver_path) ] if found_paths: driver_path = found_paths[0] opts = webdriver.FirefoxOptions() # opts.add_argument("--headless") profile = webdriver.FirefoxProfile() if not user_agent: user_agent = generate_user_agent() profile.set_preference("general.useragent.override", user_agent) driver = webdriver.Firefox( firefox_profile=profile, executable_path=driver_path, options=opts, seleniumwire_options=options, ) else: message = f"Download geckodriver from https://github.com/mozilla/geckodriver/releases/tag/v0.27.0 and paste it in {directory}" input(message) else: driver_path = os.path.join(directory, "chromedriver.exe") opts = webdriver.ChromeOptions() opts.add_argument(f"--proxy-server={opts}") driver = webdriver.Chrome(executable_path=driver_path, options=opts, seleniumwire_options=options) if not driver: input("DRIVER NOT FOUND") exit(0) driver.set_window_size(1920, 1080) browser = driver if headers: browser._client.set_header_overrides(headers=headers) return browser
def startBrowser(self): profile = webdriver.ChromeOptions() profile.add_argument('--no-sandbox') profile.add_experimental_option('excludeSwitches', ['enable-automation']) if self.headless: profile.add_argument('--headless') self.browser = webdriver.Chrome(executable_path=self.chromedriver, options=profile) self.browser.set_page_load_timeout(180) self.browser.set_window_size(1800, 1020)
def initBrowser(browserConfig, isUseSeleniumwire=False): """Init browser driver for selenium Args: browserConfig (dict): browser config isUseSeleniumwire (bool): get selenium-wire webdriver or not. Default is False. Returns: driver Raises: Examples: browserConfig normal Selenium: {'headless': True, 'pageLoadTimeout': 10} Selenium-wire: {'headless': True, 'pageLoadTimeout': 10, "seleniumwire": {"disable_capture":True}} """ webdriver = getWebDirver(isUseSeleniumwire=isUseSeleniumwire) chromeOptions = webdriver.ChromeOptions() if "headless" in browserConfig: isCurHeadless = browserConfig["headless"] if isCurHeadless: chromeOptions.add_argument('--headless') if "disableGpu" in browserConfig: isDisableGpu = browserConfig["disableGpu"] if isDisableGpu: chromeOptions.add_argument('--disable-gpu') # for debug # chromeOptions.add_experimental_option("excludeSwitches", ["enable-automation"]) # chromeOptions.add_experimental_option('useAutomationExtension', False) # chromeOptions.add_argument('window-size=1920x1080') # chromeOptions.add_argument('--window-size=1920x1080') driverConfigDict = {"options": chromeOptions} if "seleniumwire" in browserConfig: if isUseSeleniumwire: driverConfigDict["seleniumwire_options"] = browserConfig[ "seleniumwire"] else: del browserConfig["seleniumwire"] driver = webdriver.Chrome(**driverConfigDict) # driver = webdriver.Chrome(options=chromeOptions) # # for debug # driver.set_window_size(1920, 1080) if "pageLoadTimeout" in browserConfig: curPageLoadTimeout = browserConfig["pageLoadTimeout"] driver.set_page_load_timeout(curPageLoadTimeout) return driver
def initialize_driver(): option = webdriver.ChromeOptions() chrome_prefs = {} option.experimental_options["prefs"] = chrome_prefs chrome_prefs["profile.default_content_settings"] = {"images": 2} chrome_prefs["profile.managed_default_content_settings"] = {"images": 2} driver = webdriver.Chrome(options=option, seleniumwire_options=options) # driver.get("https://ipv4.icanhazip.com/") # time.sleep(2) # driver.get("https://ipv4.icanhazip.com/") # print("Proxy: ", driver.find_element_by_tag_name("body").text) return driver
def __init__(self): self.base_url = "http://moneypuck.com" opts = webdriver.ChromeOptions() sw_opts = {"disable_encoding": True} opts.add_argument("--headless") self.driver = webdriver.Chrome(options=opts, seleniumwire_options=sw_opts) with open("nhl_teams.yaml", "r") as f: self.teams = yaml.load(f, Loader=yaml.FullLoader) self.teams_inv = {v: k for k, v in self.teams.items()}
def create_headers(self,email,password): options = webdriver.ChromeOptions() # options.add_argument("--headless") with webdriver.Chrome(executable_path=ChromeDriverManager().install(),options=options) as driver: _cookie,_ua = self.login_to_fb(driver,email,password) print(_cookie) print(_ua) self.headers['cookie'] = _cookie self.headers['user-agent'] = _ua filename = email[:email.index('@')] export_file_path = self.headers_path + filename + '.json' with open(export_file_path,'w') as outputfile: json.dump(self.headers, outputfile,indent=4)
def get_driver(name: str, path: Optional[str]) -> Driver: driver_class = DRIVER_NAME_TO_CLASS[name] chrome_options = webdriver.ChromeOptions() chrome_options.headless = True args = ({ "executable_path": path, "chrome_options": chrome_options } if path else { "chrome_options": chrome_options }) return driver_class(**args)
def login(account_data: dict, initialization_headers: object, initialization_cookies: object, requests_map: dict): headers = dict() cookies = {} def interceptor(request): try: headers["x-ig-app-id"] = request.headers["X-IG-App-ID"] new = request.headers["Cookie"].split(';') for new_one in new: cooka = new_one.split('=') cookies[cooka[0].strip()] = cooka[1].strip() except AttributeError as errors_inter: logger.warning( f"The authorization process was not correct.!!! Error - {errors_inter}" ) sys.stdout.write(f"Error {errors_inter}") try: chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--window-size=10x10") driver = webdriver.Chrome(executable_path=config.PATH_TO_CHROMEDRIVER, options=chrome_options) driver.get(requests_map["main_url"]) time.sleep(5) username_field = driver.find_element_by_name( requests_map["login"]["username_field"]) password_field = driver.find_element_by_name( requests_map["login"]["password_field"]) submit_elem = driver.find_element_by_tag_name('button') username_field.send_keys(account_data["username"]) password_field.send_keys(account_data["password"]) submit_elem.click() driver.request_interceptor = interceptor request = driver.wait_for_request(requests_map["login"]["uri"]) headers["x-ig-www-claim"] = request.response.headers[ "x-ig-set-www-claim"] driver.close() set_cookies(initialization_cookies, cookies) if set_headers(initialization_headers, headers, cookies): return {"status": True} return {"status": False} except Exception as errors: logger.warning( f"The authorization process was not correct.!!! Error - {errors}") return {"status": False}
def __init__(self, username, password, brand, is_main): self.username = username self.password = password self.brand = brand self.is_main = is_main self.previousComments = open("Instagram_Comment_Names", "a") self.accounts_already_commented_on = open("Instagram_Comment_Names", "r").read().split(" ") self.comment_filter = { "post_min_number_of_comments": 0, "post_max_number_of_comments": 10000 } self.comment_pool = [] self.comments_done = 0 self.num_comments = 1 self.could_not_comment = False self.could_not_comment_popup = False self.is_liking = 1 self.liking_blocked = False options = webdriver.ChromeOptions() #chrome_options.headless = True #chrome_options.add_argument("--headless") # Runs Chrome in headless mode. #chrome_options.add_argument('--no-sandbox') # Bypass OS security model #chrome_options.add_argument('--disable-gpu') # applicable to windows os only #chrome_options.add_argument('start-maximized') # #chrome_options.add_argument('disable-infobars') #chrome_options.add_argument("--disable-extensions") #chrome_options.add_argument('--proxy-server=%s' % PROXY) user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36' options.add_argument(f'user-agent={user_agent}') #options.binary_location = os.getcwd() + "/headless-chromium" self.driver = webdriver.Chrome( executable_path=os.getcwd() + "/chromedriver", chrome_options=options, seleniumwire_options={ 'request_storage_base_dir': '/tmp', # 'proxy': { # 'https': f'https://*****:*****@{PROXY}' # } }) #self.driver.get("https://nordvpn.com/what-is-my-ip/") #print(self.driver.find_element_by_css_selector("body > div.Page.overflow-hidden.d-flex.flex-column > div:nth-child(1) > div > div > div > div > div > div > div.col-xs-12.col-sm-6.mb-7.mb-sm-0 > h3.Title.mb-6.js-ipdata-ip-address").text) self.inception_time = time.time() print("Inception:", self.inception_time) if not self.login(): sys.exit("Login failed")
def _get_default_driver(self, headless): seleniumwire_options = { 'connection_keep_alive': False, 'connection_timeout': 20 } options = webdriver.ChromeOptions() options.add_argument('window-size=1920,1080') if headless is True: options.add_argument('--disable-extensions') options.add_argument('--headless') options.add_argument('--disable-gpu') options.add_argument('--no-sandbox') return webdriver.Chrome(options=options, seleniumwire_options=seleniumwire_options)