def __init__(self, email, username, password, proxy=None): options = webdriver.ChromeOptions() options.add_experimental_option("excludeSwitches", ["enable-logging"]) self.seleniumwire_options = {'proxy': { 'http': proxy, 'https': proxy } } """if proxy: options.add_argument('--proxy-server=%s' % proxy)""" self.platform = platform.system() #Support for linux system """if self.platform == 'Windows': self.driver = webdriver.Chrome(options=options, executable_path=r"chromedriver.exe") else: self.driver = webdriver.Chrome(options=options, executable_path=r"chromedriver")""" #Webdrivermanager install chromedriver binary as per system's platform if proxy: #check if proxy is working and also update timezone as per proxy to avoid detection try: self.timezone = requests.get('http://ip-api.com/json',proxies = self.seleniumwire_options['proxy'],timeout = 10).json()['timezone'] self.tz_params = {'timezoneId': self.timezone} self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install(),seleniumwire_options=self.seleniumwire_options) except: self.timezone = None free_print(f"{Fore.LIGHTMAGENTA_EX}[!]{Style.RESET_ALL} " + 'Skipping Proxy as Connection Issue Have you entered it in the correct format?') self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install()) else: self.timezone = None self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install()) self.email= email self.username = username self.password = password
def create_chromedriver(self): """ Since we have many functions we can perform we consolidate chromedriver code here. """ # set up options object chrome_options = Options() # if we have chrome binary set it up if self.chrome_binary_path: chrome_options.binary_location = self.chrome_binary_path # live dangerously if self.allow_insecure: chrome_options.add_argument('--allow-running-insecure-content') # thank god for this option chrome_options.add_argument('--mute-audio') # if we are headless we also mix up window size a bit if self.headless: chrome_options.add_argument('headless') chrome_options.add_argument('disable-gpu') window_x = random.randrange(1050, 1920) window_y = random.randrange(900, 1080) chrome_options.add_argument('window-size=%sx%s' % (window_x, window_y)) # if we have a ua set it here if self.ua: chrome_options.add_argument('user-agent=' + self.ua) # attempt to start driver, fail gracefull otherwise try: # if we have chromedriver path set it up if self.chromedriver_path: driver = webdriver.Chrome(self.chromedriver_path, desired_capabilities={ 'loggingPrefs': { 'performance': 'ALL' } }, chrome_options=chrome_options) else: driver = webdriver.Chrome( desired_capabilities={ 'loggingPrefs': { 'performance': 'ALL' } }, chrome_options=chrome_options, ) except: return None # allow one minute before we kill it, seperate from browser_wait driver.set_page_load_timeout(self.page_timeout_seconds) return driver
def __buildDriver__(self, driver_options): # Ubuntu if (os.name == 'posix'): chromedriver = 'chromedriver' # Windows if (os.name == 'nt'): chromedriver = 'chromedriver.exe' if chromedriver: if config.DRIVER_NAME == 'Chrome': logger.info("Using Chrome Driver ...") options = webdriver.ChromeOptions() options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) options.add_experimental_option('w3c', False) for driver_option in driver_options: # if "--proxy-server" in driver_option: # print(f"\nADDING PROXY: [{driver_option}]\n") options.add_argument(driver_option) capabilities = None if config.PROXY: # print(f"\nADDING PROXY: [{config.PROXY}]\n") # prox = Proxy() # prox.proxy_type = ProxyType.MANUAL # prox.http_proxy = config.PROXY # # prox.socks_proxy = config.PROXY # prox.ssl_proxy = config.PROXY capabilities = webdriver.DesiredCapabilities.CHROME capabilities['loggingPrefs'] = {'performance': 'ALL'} # prox.add_to_capabilities(capabilities) if capabilities: self.driver = webdriver.Chrome( desired_capabilities=capabilities, options=options) # self.driver = config.DRIVER_NAME(desired_capabilities=capabilities, options=options) else: self.driver = webdriver.Chrome(chromedriver, options=options) # self.driver = config.DRIVER_NAME(chromedriver, options=options) else: logger.info("Using Firefox Driver ...") self.driver = webdriver.Firefox() self.min_wait = WebDriverWait(self.driver, 5) self.max_wait = WebDriverWait(self.driver, 20) self.els_css = self.driver.find_elements_by_css_selector self.el_css = self.driver.find_element_by_css_selector self.els_xpath = self.driver.find_elements_by_xpath self.el_xpath = self.driver.find_element_by_xpath self.driver.set_script_timeout(30) return self.driver
def set_driver_for_browser(self, browser_name): """expects browser name and returns a driver instance""" # if browser is suppose to be chrome if browser_name.lower() == "chrome": browser_option = ChromeOptions() # automatically installs chromedriver and initialize it and returns the instance if self.proxy is not None: options = { 'https': 'https://{}'.format(self.proxy.replace(" ", "")), 'http': 'http://{}'.format(self.proxy.replace(" ", "")), 'no_proxy': 'localhost, 127.0.0.1' } print("Using: {}".format(self.proxy)) return webdriver.Chrome(executable_path=ChromeDriverManager().install(), options=self.set_properties(browser_option), seleniumwire_options=options) return webdriver.Chrome(executable_path=ChromeDriverManager().install(), options=self.set_properties(browser_option)) elif browser_name.lower() == "firefox": browser_option = FirefoxOptions() if self.proxy is not None: options = { 'https': 'https://{}'.format(self.proxy.replace(" ", "")), 'http': 'http://{}'.format(self.proxy.replace(" ", "")), 'no_proxy': 'localhost, 127.0.0.1' } print("Using: {}".format(self.proxy)) return webdriver.Firefox(executable_path=GeckoDriverManager().install(), options=self.set_properties(browser_option), seleniumwire_options=options) # automatically installs geckodriver and initialize it and returns the instance return webdriver.Firefox(executable_path=GeckoDriverManager().install(), options=self.set_properties(browser_option)) else: # if browser_name is not chrome neither firefox than raise an exception raise Exception("Browser not supported!")
def __init__(self, proxy_class, price_class): """Создаем экземпляр бота""" Thread.__init__(self) if settings.DEBUG: print('CREATE BOT') self.proxies = proxy_class self.price = price_class self.no_proxy = False self.use_proxy = 1 if self.use_proxy == 2: self.proxy = '' else: self.proxy, clean_proxy = self.proxies.get_new_proxy() """Подключаем настройки прокси к экземпляру бота""" options = {'proxy': self.proxy} wind_width = random.randint(1000, 1920) wind_height = random.randint(750, 1080) with open( f'{settings.MEDIA_DIR}/user-agents/user_agents_for_chrome_pk.txt', 'r', encoding='utf-8') as f: user_agent = random.choice([line.strip() for line in f]) self.options = webdriver.ChromeOptions() self.options.add_argument("ignore-certificate-errors") self.options.add_argument(f'window-size={wind_width},{wind_height}') self.options.add_argument("--disable-blink-features") self.options.add_argument( "--disable-blink-features=AutomationControlled") self.options.add_argument("--headless") self.options.add_argument('--disable-extensions') self.options.add_argument('--profile-directory=Default') """Добавялем юзер агент из списка""" self.options.add_argument(f"user-agent={user_agent}") """Создаем экземпляр браузера""" if self.use_proxy == 1: self.driver = webdriver.Chrome( options=self.options, executable_path=f'{settings.MEDIA_DIR}/drivers/chromedriver', seleniumwire_options=options, ) else: self.driver = webdriver.Chrome( options=self.options, executable_path=f'{settings.MEDIA_DIR}/drivers/chromedriver', ) """Создаем необходимые переменные для работы""" self.reboot = False self.search_status = False self.step = 0 self.target = []
def get_webdriver(headless=False, executable_path=r'./chromedriver.exe'): if headless: options = Options() options.add_argument("--headless") driver = wire_webdriver.Chrome(options=options) else: #driver = webdriver.Chrome('C:\\Users\\caioc\\Desktop\\scripts\\chromedriver') driver = wire_webdriver.Chrome(executable_path=executable_path) return driver
def select_driver(browser): if browser == "Chrome": return webdriver.Chrome(executable_path='/usr/local/bin/chromedriver', seleniumwire_options={'verify_ssl': False}) # return webdriver.Chrome(executable_path='/usr/local/bin/chromedriver') elif browser == "Firefox": return webdriver.Firefox() else: return webdriver.Chrome(executable_path='/usr/local/bin/chromedriver')
def getFederatedRequestParams(username, password, samlUrl): import selenium from seleniumwire import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.chrome.options import Options import time from urllib.parse import unquote from urllib.parse import quote # add some headless options options = Options() options.headless = True options.add_experimental_option("excludeSwitches",["ignore-certificate-errors"]) # using chrome to access web pages driver = webdriver.Chrome(chrome_options=options) # Open federated authentication webpage that returns the needed saml response driver.get(samlUrl) # select the username box when it to loads wait = WebDriverWait(driver, 10) id_box = wait.until(EC.element_to_be_clickable((By.ID, 'userNameInput'))) # enter username id_box.send_keys(username) # select password box pass_box = driver.find_element_by_id('passwordInput') # enter password pass_box.send_keys(password) # select Sign in box signIn_button = driver.find_element_by_id('submitButton') # click Sign in button signIn_button.click() # find the proper response header and use it to get the bearer token for request in driver.requests: if request.response and request.path == "https://anypoint.mulesoft.com/accounts/login/receive-id": # convert from binary to string and strip leading text samlResponse = unquote(str(request.body,'utf-8'))[13:] # get the bearer token url = 'https://anypoint.mulesoft.com/accounts/login/receive-id' body = '{"SAMLResponse": "%s"}' % samlResponse headers = { 'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest' } # close the chrome browser instance driver.quit() return url, headers, body
def __init__(self, url): self.url = url #self.options = webdriver.ChromeOptions() #self.options.add_argument('headless') # для открытия headless-браузера #self.driver= webdriver.Chrome(options=self.options) self.driver = webdriver.Chrome() self.navigate()
def get_chrome_driver(driver_path): """ Instantiates a Chrome webdriver object Args: driver_path: path to the Chrome driver Returns: webdriver: returns a webdriver object """ options = webdriver.ChromeOptions() options.page_load_strategy = 'normal' options.add_argument("--disable-logging") options.add_argument("--disable-login-animations") options.add_argument("--disable-notifications") options.add_argument("--disable-default-apps") options.add_argument("--disable-extensions") options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) options.add_argument("--disable-dev-shm-usage") options.add_argument("--disable-browser-side-navigation") options.add_argument("--headless") options.add_argument("--hide-scrollbars") options.add_argument('--log-level 3') options.add_argument("--incognito") options.add_argument("--no-zygote") options.add_argument('--disable-gpu') options.add_argument('--no-sandbox') options.add_argument('--disable-infobars') driver = webdriver.Chrome(driver_path, options=options) driver.set_page_load_timeout(PAGE_LOAD_TIMEOUT_SEC) return driver
def post(self, request): try: if request.is_ajax(): command = request.POST.get('command') if command == 'start': url = request.POST.get('url') scopes = request.POST.getlist('scopes[]') options = { 'cleanup_onexit': False, 'connection_timeout': None # Never timeout } # Download and install Chrome web driver global driver driver = webdriver.Chrome( ChromeDriverManager().install(), seleniumwire_options=options) # Specifying driver scopes driver.scopes = scopes # Opening up the URL address driver.get(url) #p1 = multiprocessing.Process(target=mytest) # p1.start() return JsonResponse({'status': 'success'}, status=200) else: return JsonResponse({'status': 'success'}, status=200) except: return JsonResponse({'response': 'error'}, status=200)
def get_hangouts_tokens(cookies, driverpath): ''' gets auth and hangout token ''' tmprinter = TMPrinter() chrome_options = get_chrome_options_args(config.headless) options = { 'connection_timeout': None # Never timeout, otherwise it floods errors } tmprinter.out("Starting browser...") driver = webdriver.Chrome(executable_path=driverpath, seleniumwire_options=options, options=chrome_options) driver.header_overrides = config.headers tmprinter.out("Setting cookies...") driver.get("https://hangouts.google.com/robots.txt") for k, v in cookies.items(): driver.add_cookie({'name': k, 'value': v}) tmprinter.out("Fetching Hangouts homepage...") driver.get("https://hangouts.google.com") tmprinter.out("Waiting for the /v2/people/me/blockedPeople request, it " "can takes a few minutes...") req = driver.wait_for_request('/v2/people/me/blockedPeople', timeout=120) tmprinter.out("Request found !") driver.close() tmprinter.out("") auth_token = req.headers["Authorization"] hangouts_token = req.url.split("key=")[1] return (auth_token, hangouts_token)
def load_browser(): global driver global ublock_guid chrome_options = webdriver.ChromeOptions() # chrome_options.binary_location = "/applications/developer/google\ chrome.app/Contents/MacOS/Google\ Chrome" # chrome_options.add_argument("--disable-web-security") # messes up ublock # chrome_options.add_argument('--headless') chrome_options.add_extension('ubo_1_30_4_0.crx') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-gpu') chrome_options.add_argument("--ignore-certificate-errors") chrome_options.add_argument("--enable-javascript") chrome_options.add_argument("--disable-chrome-google-url-tracking-client") chrome_options.add_argument("--safebrowsing-disable-download-protection") chrome_options.add_argument("--disable-domain-reliability") chrome_options.add_argument("--allow-running-insecure-content") chrome_options.add_argument( "--unsafely-treat-insecure-origin-as-secure=http://host:port") chrome_options.add_argument( '--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36' ) driver = webdriver.Chrome(executable_path="./chromedriver", options=chrome_options) driver.header_overrides = {'Referer': 'com.google.android.gm'} # display = Display(visible=0, backend="xephyr", size=(800, 600)) # display.start() extension_uri = "chrome-extension://cjpalhdlnbpafiamejdnhcphjbkeiagm/logger-ui.html?popup=1#_" driver.implicitly_wait(page_delay) driver.get(extension_uri) ublock_guid = driver.current_window_handle
def test_proxy(proxy,url,page_title,use_request=False): host = proxy[0] port = proxy[1] username = proxy[2] password = proxy[3] if use_request == True: proxy_argument = {'https': f'https://{username}:{password}@{host}:{port}'} try: print("Testing proxy") r = requests.get(f'https://{url}', proxies=proxy_argument, timeout=3.5) print('Found working proxy') return proxy except requests.exceptions.ProxyError: print(f'{proxy} Banned on {url}') else: options = { 'proxy': { 'https': f'https://{username}:{password}@{host}:{port}', } } browser = webdriver.Chrome("/Users/CreeperFire/Desktop/PythonT/chromedriver", seleniumwire_options=options) try: browser.get(url) browser.implicitly_wait(5) if page_title in browser.title == True: return proxy except OSError or selenium.common.exceptions.TimeoutException: print(proxy,"banned") browser.close()
def __init__(self): self.browser = webdriver.Chrome(chromedriver, options=chrome_options, seleniumwire_options=proxy_options) self.telegram = TelegramBot() self.max_likes = 20 self.likes_count = 0
def get_driver(*args, **kwargs): options = Options() options.headless = False options.add_argument("--window-size=1920,1080") options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') options.add_argument('--disable-crash-reporter') options.add_argument('--disable-logging') options.add_argument('--log-level=3') options.add_argument('--ignore-ssl-errors=yes') options.add_argument('--ignore-certificate-errors') options.add_argument( '--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36' ) if platform.system() == 'Linux': DRIVER_PATH = 'chromedriver' elif platform.system() == "Darwin": DRIVER_PATH = (pathlib.Path(__file__).parent.parent / 'chromedriver').resolve() else: log.error('Unsupported OS') exit(0) # TODO find a good proxy to use so that this works from Azure proxy_options = {'auto_config': False, 'disable_capture': True} driver = webdriver.Chrome(options=options, executable_path=DRIVER_PATH, seleniumwire_options=proxy_options) yield driver driver.close() driver.quit()
def init_chrome_driver(driver_path=None) -> WebDriver: """ Initialize a Chrome webdriver with custom headers support Assumes Chrome Driver path executable is set by env :: CHROME_DRIVER_PATH, defaults to /usr/local/bin """ if driver_path is None: driver_path = os.environ.get('CHROME_DRIVER_PATH', '/usr/local/bin/chromedriver') logger.info(f"Chrome Driver path at '{driver_path}'") options = Options() options.add_argument("--headless") options.add_argument("--window-size=1920x1080") options.add_argument('--log-level 3') logger.debug(f"Chrome options: '{options}'") logger.debug(json.dumps(options.to_capabilities(), indent=4)) driver = webdriver.Chrome(chrome_options=options, executable_path=driver_path) return driver
def __init__( self, options=None, user_agent=None, wait_time=2, fullscreen=False, incognito=False, headless=False, **kwargs ): options = Options() if options is None else options if user_agent is not None: options.add_argument("--user-agent=" + user_agent) if incognito: options.add_argument("--incognito") if fullscreen: options.add_argument("--kiosk") if headless: options.add_argument("--headless") options.add_argument("--disable-gpu") self.driver = webdriver.Chrome(options=options, **kwargs) self.element_class = WebDriverElement self._cookie_manager = CookieManager(self.driver) super().__init__(wait_time)
def crawlWebsite(): print('[ INFO ] ' + 'Crawling from ' + base_url) driver = webdriver.Chrome('/usr/local/bin/chromedriver', seleniumwire_options=options, options=chrome_options) driver.implicitly_wait(5) driver.get(base_url) more_button = driver.find_elements_by_xpath(MUAT_LAINNYA_XPATH) btn_counter = 0 while (len(more_button) > 0) and btn_counter < 100: more_button[0].click() btn_counter += 1 print('[ INFO ] Clicked MUAT LAINNYA! Number of clicks: ' + str(btn_counter)) time.sleep(2) more_button = driver.find_elements_by_xpath(MUAT_LAINNYA_XPATH) print('[ INFO ] No more MUAT LAINNYA button. Oh boy time to parse!') page = soup(driver.page_source, 'html.parser') car_tags = page.find_all(class_='EIR5N') car_data = [] print('[ INFO ] Total link: ' + str(len(car_tags))) for cars in car_tags: car_link = cars.find('a', href=True) car_url = car_url_prefix + car_link['href'] value_dict = crawlCarData(car_url) if (bool(value_dict)): car_data.append(value_dict) percentage = str(int( (len(car_data) / len(car_tags)) * 100)).rjust(3) print('[ ' + percentage + '% ] ' + 'Crawled from ' + car_url) time.sleep(1) driver.close() print('[ INFO ] Crawling finished. Total data crawled: ' + str(len(car_data))) return car_data
def initiate_driver(url): global driver #options = webdriver.ChromeOptions() #options.add_experimental_option('debuggerAddress', 'localhost:9014') driver = webdriver.Chrome(executable_path="chromedriver.exe", seleniumwire_options={'port': 9014}) driver.get(url)
def driver(self): if self._driver is None: options = self.options() scopes = self.scopes() self._driver = webdriver.Chrome(options=options) self._driver.scopes = scopes return self._driver
def initialize_web_driver(self): # Add additional Options to the webdriver chrome_options = ChromeOptions() # add the argument and make the browser Headless. if self.headless_browser: chrome_options.add_argument("--headless") """ chrome_options.add_argument("--ignore-certificate-errors") chrome_options.add_argument("--incognito") chrome_options.add_argument("--kiosk") """ # set options """ chrome_options.add_argument("--window-size=1920,1080") chrome_options.add_argument("--disable-gpu") chrome_options.add_argument("--disable-extensions") chrome_options.add_argument("--proxy-server='direct://'") chrome_options.add_argument("--proxy-bypass-list=*") chrome_options.add_argument("--start-maximized") """ driver = webdriver.Chrome( executable_path='E:/chromedriver_win32/chromedriver.exe', options=chrome_options) print('got driver') return driver
def __init__(self, frontier_lock, access_time_lock): self.USER_AGENT = "fri-wier-agmsak" self.WEB_DRIVER_LOCATION = "chromedriver" self.wait_for = 5 self.delay = 5 self.frontier_lock = frontier_lock self.access_time_lock = access_time_lock self.DEBUG = False self.site_table = tables.SiteTable() self.page_table = tables.PageTable() self.image_table = tables.ImageTable() self.ip_table = tables.SiteIPAddrTable() self.link_table = tables.LinkTable() self.pagedata_table = tables.PageDataTable() self.current_page_id = None self.current_url = None self.processing_page = None options = Options() options.add_argument("--headless") options.add_argument("user-agent=" + self.USER_AGENT) self.driver = webdriver.Chrome(self.WEB_DRIVER_LOCATION, options=options)
def init_driver(self): # settings selenium driver_options = webdriver.ChromeOptions() if self.browser_executable_path: driver_options.binary_location = self.browser_executable_path # # switch user_agent, proxy # if hasattr(self, 'agent'): # driver_options.add_argument(f"--user-agent='{next(self.agent)}'") # if hasattr(self, 'proxy'): # driver_options.add_argument(f"--proxy-server='{next(self.proxy)}'") # add SELENIUM_DRIVER_ARGUMENTS options for argument in self.driver_arguments: driver_options.add_argument(argument) # add experimental option: on/off image and notify chrome_prefs = {"disk-cache-size": 4096} if not self.driver_load_img: chrome_prefs.update( { "profile.default_content_settings": {"images": 2}, "profile.managed_default_content_settings": {"images": 2} } ) if self.driver_disable_notify: chrome_prefs.update( {"profile.default_content_setting_values.notifications": 2} ) driver_options.add_experimental_option("prefs", chrome_prefs) self.driver = webdriver.Chrome( executable_path=self.driver_executable_path, options=driver_options )
def get_info(link): reviews = [] options = 1 driver = 1 try: options = webdriver.ChromeOptions() options.add_argument('--ignore-certificate-errors') options.add_argument('--incognito') options.add_argument('--headless') driver = webdriver.Chrome(executable_path=binary_path, chrome_options=options) driver.get(link) wait = WebDriverWait(driver,120) wait.until(wait_requests) except TimeoutException: print("Failed to load search bar at www.google.com") except Exception as e: print(e) finally: for request in driver.requests: if request.response and request.response.headers['Content-Type'] == 'application/json' and request.path == 'https://www.tripadvisor.es/data/graphql/batched': response = json.loads(request.response.body) for r in response: if 'locations' in r['data']: try: for location in r['data']['locations']: if 'reviewListPage' in location: for review in location['reviewListPage']['reviews']: r = Review(review['id'], review['userId'], review['title'], review['text'], review['language']) reviews.append(r) except Exception as e: "" driver.quit() return reviews
def initialize_driver(headless=True): print("[.] Initialising chromedriver...") chrome_options = Options() if (headless): chrome_options.add_argument("--headless") chrome_options.add_argument("window-size=1920,1080") chrome_options.add_argument("--log-level=3"); chrome_options.add_argument("--silent"); chrome_options.add_argument("--disable-logging") # this allows selenium to accept cookies with a non-int64 'expiry' value chrome_options.add_experimental_option("w3c", False) # check OS to pick the correct driver current_system = platform.system() if (current_system == 'Windows'): driver_path = os.path.join(path, "bin", "chromedriver.exe") elif (current_system == 'Darwin'): driver_path = os.path.join(path, "bin", "chromedriver") else: print('[!] Unsupported OS.') sys.exit() driver = webdriver.Chrome( executable_path=driver_path, service_log_path=os.path.join(path, "webdrive.log"), # Don't verify self-signed cert, should help with 502 errors (https://github.com/wkeeling/selenium-wire/issues/55) # seleniumwire_options={'verify_ssl': False}, options=chrome_options) return driver;
def _authoriz_upd(self): url = "https://app.zakupki360.ru/login" chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--headless') with webdriver.Chrome(executable_path="./chromedriver", options=chrome_options) as driver: driver.implicitly_wait(10) driver.get(url) driver.find_element_by_id("mat-input-0").send_keys( config.authoriz['login']) driver.find_element_by_id("mat-input-1").send_keys( config.authoriz['password']) driver.find_element_by_tag_name('button').click() driver.find_element_by_class_name("items") resp = tuple({ req.headers['Authorization'] for req in driver.requests if req.response }) try: key = resp[0] if resp[0] else resp[1] with open("auth_key", "w") as f_out: f_out.write(key) return key except Exception as e: print(resp) print(e)
def __init__(self, *args, **kwargs): chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--headless') chrome_options.add_argument('connection_keep_alive=False') chrome_options.add_argument( 'disable-blink-features=AutomationControlled') chrome_options.add_argument( "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:79.0) Gecko/20100101 Firefox/79.0" ) options = { 'backend': 'mitmproxy', 'disable_encoding': True # Tell the server not to compress the response # Set the backend to mitmproxy } self.driver = webdriver.Chrome('chromedriver', options=chrome_options, seleniumwire_options=options) self.driver.maximize_window() self.driver.execute_cdp_cmd( "Page.addScriptToEvaluateOnNewDocument", { "source": """ Object.defineProperty(navigator, 'webdriver', { get: () => undefined }) """ })
def _start_chrome_driver(self) -> webdriver: """Using Selenium start the google chrome browser headless. All the browser requests and responses(har_fit data) will be recorded using a BrowsermobProxy proxy server. :return: Google chrome driver object. """ chrome_options = webdriver.ChromeOptions() prefs = {"profile.default_content_setting_values.notifications": 2} chrome_options.add_experimental_option("prefs", prefs) chrome_options.set_capability( 'proxy', { 'httpProxy': f'{self.proxy.proxy}', 'noProxy': '', 'proxyType': 'manual', 'sslProxy': f'{self.proxy.proxy}' }) # chrome_options.add_argument("--headless") driver = webdriver.Chrome(chrome_options=chrome_options) driver.set_page_load_timeout(999) driver.delete_all_cookies() return driver
def __init__(self, driver_path='/usr/local/bin/chromedriver', headless=True): self.options = Options() self.options.headless = headless self.options.add_argument('--no-sandbox') self.options.add_argument('--disable-dev-shm-usage') self.driver = webdriver.Chrome(driver_path, chrome_options=self.options) self.html = None