def initBrowser(): print("\nInitializing browser...", end="") if BROWSER_DRIVER.lower().startswith("chrome"): chromeOptions = webdriver.ChromeOptions() chromeOptions.add_argument("--disable-infobars") chromeOptions.add_argument("--disable-gpu") chromeOptions.add_argument("--disable-extensions") chromeOptions.add_argument("--window-size=800,800") chromeOptions.add_argument("--incognito") chromeOptions.add_experimental_option('excludeSwitches', ['enable-logging']) chromeOptions.add_experimental_option("prefs", {"profile.default_content_setting_values.media_stream_mic": 2, "profile.default_content_setting_values.media_stream_camera": 2, "profile.default_content_setting_values.notifications": 2 }) if BROWSER_DRIVER.lower().endswith(".exe"): driver = webdriver.Chrome(executable_path=BROWSER_DRIVER, options=chromeOptions) else: servicePath = Service(BROWSER_DRIVER) driver = webdriver.Chrome(service=servicePath, options=chromeOptions) elif BROWSER_DRIVER.lower().startswith("firefox"): firefoxOptions = webdriver.FirefoxOptions() firefoxOptions.add_argument("--width=800"), firefoxOptions.add_argument("--height=800") # firefoxOptions.headless = True firefoxOptions.set_preference("layers.acceleration.disabled", True) firefoxOptions.set_preference("browser.privatebrowsing.autostart", True) firefoxOptions.set_preference("permissions.default.microphone", 2) firefoxOptions.set_preference("permissions.default.camera", 2) if BROWSER_DRIVER.lower().endswith(".exe"): driver = webdriver.Firefox(executable_path=BROWSER_DRIVER, options=firefoxOptions) else: servicePath = Service(BROWSER_DRIVER) driver = webdriver.Firefox(service=servicePath, options=firefoxOptions) print(colored(" Success!", "green")) return(driver)
def open_instagram(self): if os.name == "nt": # if its windows try: self.brows = Chrome(executable_path=chrome_path, chrome_options=headless_for_chrome()) except WebDriverException: print(its_not_ok + "Cannot find Chrome binary...\nTrying Firefox") self.brows = Firefox(service=Service(firefox_path), options=headles_for_firefox()) else: try: self.brows = Firefox(service=Service(firefox_path), options=headles_for_firefox()) except WebDriverException: print(its_not_ok + "Cannot find gecko...\nTrying install") install_to_os() print( its_ok + "Installed Successfully Again do you want to headles or not ?" ) self.brows = Firefox(service=Service(firefox_path), options=headles_for_firefox()) self.brows.maximize_window() self.brows.implicitly_wait(20) self.brows.get("https://www.instagram.com/") self.find_username = self.brows.find_element( By.NAME, "username").send_keys(self.username) self.find_passwd = self.brows.find_element( By.NAME, "password").send_keys(self.passwd)
def setup_driver() -> webdriver: Options = options.Options() Options.headless = True driver = webdriver.Firefox(options=Options, service=Service(GeckoDriverManager().install())) return driver
def setUpfirefox(self): """Product name: Firefox Nightly Product version: 71.0a1 Firefox can run headless with sendkeys. (Firefox handler is called GeckoDriver) Note about firefox driver on MacOS: if it fails to load there's a simple one-time workaround: https://firefox-source-docs.mozilla.org/testing/geckodriver/Notarization.html""" options = Options() options.headless = True try: if self.running_platform == "Darwin": # If it's a mac, then use the old API code regardless of Selenium version handler = webdriver.Firefox(options=options, executable_path=self.handler_path + 'geckodriver') elif self.running_platform == "Windows" and WebPage.selenium_ver == "4": # If it's Windows, then check selenium version service = Service( Path(self.handler_path + 'geckodriver.exe' )) # Specify the custom path (new for Selenium 4) handler = webdriver.Firefox(options=options, service=service) elif self.running_platform == "Windows": handler = webdriver.Firefox( options=options, executable_path=Path(self.handler_path + 'geckodriver.exe')) else: # In case it's Unix handler = webdriver.Firefox( options=options, executable_path=Path(self.handler_path + 'geckodriver')) logging.info( f"{datetime.now(tz=None)} Info Firefox browser handler found") except (WebDriverException): logging.info( f"{datetime.now(tz=None)} Warning Firefox browser handler not found or failed to launch." ) handler = None return handler
def firefox_get_cookies(url): """ :param url: 请求连接 :return: """ c_service = Service('geckodriver') driver = any try: c_service.command_line_args() c_service.start() firefox_options = Options() # 不启动界面显示- linux下命令行模式必须启用 firefox_options.add_argument('-headless') driver = Firefox(options=firefox_options) driver.get(url) # 第一次请求浏览器一般无法显示cookie # 等待第一次加载完成 time.sleep(2) # 刷新 driver.refresh() # 等待第二次加载完成 time.sleep(2) return driver.get_cookies() except Exception as e: print(e) finally: driver.quit() c_service.stop()
def SetUpIE(self): """Product name: Selenium WebDriver Product version: 2.42.0.0 IE does not have support for a headless mode IE has some other gotchas, too, which I posted in my blog. See https://speakingpython.blogspot.com/2020/07/working-with-selenium-webdriver-in.html """ try: if self.selenium_ver == "4": # for IE, we use the IEDriverServer which might be why it redirects (see log) service = Service( Path(self.handler_path + 'IEDriverServer.exe') ) # Specify the custom path (new for Selenium 4) handler = webdriver.Ie(service=service) logging.info( f"{datetime.now(tz=None)} Info {self.browse} Finished handler setup" ) else: handler = webdriver.Ie( executable_path=Path(self.handler_path + 'IEDriverServer.exe')) handler.maximize_window() logging.info( f"{datetime.now(tz=None)} Info {self.browse} browser handler found" ) except (WebDriverException): logging.info( f"{datetime.now(tz=None)} Warning {self.browse} browser handler not found or failed to launch." ) handler = None return handler
def create_driver_instance(self, options: Options, geckodriver_path: str): """ Create a WebDriver instance. Isolate retrying code here to address "no connection can be made" error. :param options: the FirefoxOptions for the browser. :param geckodriver_path: the path to the firefox executable. the icomoon.zip to. """ retries = SeleniumRunner.MAX_RETRY finished = False driver = None err_msgs = [] # keep for logging purposes while not finished and retries > 0: try: # order matters, don't change the lines below finished = True # signal we are done in case we are actually done # customize the local server service = None # first retry: use 8080 # else: random if retries == SeleniumRunner.MAX_RETRY: service = Service(executable_path=geckodriver_path, port=8080) else: service = Service(executable_path=geckodriver_path) driver = WebDriver(options=options, service=service) except SeleniumTimeoutException as e: # retry. This is intended to catch "no connection could be made" error retries -= 1 finished = False # flip the var so we can retry msg = f"Retry {retries}/{SeleniumRunner.MAX_RETRY} SeleniumTimeoutException: {e.msg}" print(msg) err_msgs.append(msg) except Exception as e: # anything else: unsure if retry works. Just end the retry msg = f"Retry {retries}/{SeleniumRunner.MAX_RETRY} Exception: {e}" err_msgs.append(msg) print(msg) break if driver is not None: return driver err_msg_formatted = '\n'.join(reversed(err_msgs)) msg = f"Unable to create WebDriver Instance:\n{err_msg_formatted}" raise Exception(msg)
def _default_browser(): display = Display(visible=False, size=(800, 600)) display.start() firefox_options = Options() firefox_options.headless = True firefox_options.set_preference("gfx.webrender.all", True) service = Service(Constants.FIREFOX_BIN_PATH, log_path=Constants.LOG_FILE_PATH) return webdriver.Firefox(service=service, options=firefox_options)
def __init__(self, _client_key, _headless): self.options = Options() self.options.headless = _headless self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0" self.captcha = RecaptchaV2Task(_client_key) self.browser = webdriver.Firefox(service=Service( GeckoDriverManager().install()), options=self.options) self.website_url = "https://www.google.com/recaptcha/api2/demo"
def __init__(self, _client_key, _headless): self.options = Options() self.options.headless = _headless self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0" self.captcha = HCaptchaTask(_client_key) self.browser = webdriver.Firefox(service=Service( GeckoDriverManager().install()), options=self.options) self.website_url = "http://hcaptcha.jawne.info.pl/" self.expected = "Your request have submitted successfully"
def open_instagram(self): if os.name == "nt": try: self.brows = Chrome(executable_path=chrome_path, chrome_options=headless_for_chrome()) except WebDriverException: print("Cannot find Chrome binary....\nTrying Firefox") self.brows = Firefox(service=Service(firefox_path), options=headles_for_firefox()) else: self.brows = Firefox(service=Service(firefox_path), options=headles_for_firefox()) self.brows.get("https://www.instagram.com/") self.brows.implicitly_wait(20) self.find_username = self.brows.find_element_by_name( "username").send_keys(self.username) self.find_passwd = self.brows.find_element_by_name( "password").send_keys(self.passwd)
def __init__(self, _client_key, _headless): self.options = Options() self.options.headless = _headless self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0" self.captcha = FuncaptchaTask(_client_key) self.browser = webdriver.Firefox(service=Service( GeckoDriverManager().install()), options=self.options) self.website_url = "https://client-demo.arkoselabs.com/solo-animals" self.expected = "Solved!"
def initialize_webdriver(): # Run webdriver with a new user agent each time it starts. service = Service(log_path="/dev/null") options = Options() options.headless = True options.set_preference("general.useragent.override", get_user_agent()) options.set_preference("dom.webdriver.enabled", False) options.set_preference("useAutomationExtension", False) driver = webdriver.Firefox(service=service, options=options) driver.maximize_window() return driver
def test_command_line_args(): service = Service("geckodriver", service_args=["--log", "trace"]) found = False args = service.command_line_args() for idx in range(len(args) - 1): if args[idx] == "--log" and args[idx + 1] == "trace": found = True break assert found, "Provided arguments do not exist in array"
def setUpClass(cls) -> None: super().setUpClass() if settings.SKIP_SELENIUM_TESTS: return if settings.VIRTUAL_DISPLAY: cls.vdisplay = Display(visible=False, size=(1024, 768)) cls.vdisplay.start() service = Service(settings.GECKODRIVER_PATH, log_path=settings.GECKODRIVER_LOG_PATH) cls.selenium = Firefox(service=service)
def create_driver_instance(self, options: Options, geckodriver_path: str): """ Create a WebDriver instance. Isolate retrying code here to address "no connection can be made" error. :param options: the FirefoxOptions for the browser. :param geckodriver_path: the path to the firefox executable. the icomoon.zip to. """ driver = None err_msgs = [] # keep for logging purposes for i in range(SeleniumRunner.MAX_RETRY): try: # customize the local server service = None # first try: use 8080 # else: random if i == 0: service = Service(executable_path=geckodriver_path, port=8080) else: service = Service(executable_path=geckodriver_path) driver = WebDriver(options=options, service=service) except Exception as e: # retry. This is intended to catch "no connection could be made" error # anything else: unsure if retry works. Still retry msg = f"Retry {i + 1}/{SeleniumRunner.MAX_RETRY} Exception: {e}" err_msgs.append(msg) print(msg, file=self.log_output) else: # works fine break else: # out of retries # handle situation when we can't make a driver err_msg_formatted = '\n'.join(reversed(err_msgs)) msg = f"Unable to create WebDriver Instance:\n{err_msg_formatted}" raise Exception(msg) return driver
class GenParserSel: service = Service(executable_path="/Users/mattkohl/Downloads/geckodriver") options = Options() options.binary_location = "/Applications/Firefox Developer Edition.app/Contents/MacOS/firefox-bin" options.headless = True options.add_argument("--window-size=1920,1200") driver: WebDriver = webdriver.Firefox(service=service, options=options) cipher = "Lyrics for this song have yet to be released" techniques = [[By.ID, "lyrics-root"], [By.CLASS_NAME, "lyrics"]] @staticmethod def download(urls: List[str]) -> Tuple[Optional[str], str]: _techniques = GenParserSel.techniques.copy() def parse(bys: List[List]): by = bys.pop() _lyrics = None try: print(f"Attempting extraction using {by}") element = GenParserSel.driver.find_element(*by) _lyrics = element.text _lyrics.rstrip("Embed") _lyrics = _lyrics if GenParserSel.cipher not in _lyrics else None except Exception as _e: print(f"Unsuccessful lyric extraction using {by}: {_e}") if len(bys) >= 1: return parse(bys) else: return None return _lyrics assert len(urls) > 0 url = urls.pop() try: print(f"Attempting download using {url}") GenParserSel.driver.get(url) assert ("Burrr!" not in GenParserSel.driver.title) except Exception as e: print(f"Failed to download lyrics from: {url}") if len(urls) >= 1: return GenParserSel.download(urls) else: return None, url lyrics = parse(_techniques) return lyrics, url
def __init__(self, firefox_profile=None, firefox_binary=None, timeout=30, capabilities=None, proxy=None, executable_path="wires", firefox_options=None): capabilities = capabilities or DesiredCapabilities.FIREFOX.copy() self.profile = firefox_profile or FirefoxProfile() self.profile.native_events_enabled = ( self.NATIVE_EVENTS_ALLOWED and self.profile.native_events_enabled) self.binary = firefox_binary or capabilities.get( "binary", FirefoxBinary()) self.options = firefox_options or Options() self.options.binary_location = self.binary if isinstance( self.binary, basestring) else self.binary._start_cmd self.options.profile = self.profile capabilities.update(self.options.to_capabilities()) # marionette if capabilities.get("marionette"): self.service = Service(executable_path, firefox_binary=self.options.binary_location) self.service.start() executor = FirefoxRemoteConnection( remote_server_addr=self.service.service_url) RemoteWebDriver.__init__(self, command_executor=executor, desired_capabilities=capabilities, keep_alive=True) else: # Oh well... sometimes the old way is the best way. if proxy is not None: proxy.add_to_capabilities(capabilities) executor = ExtensionConnection("127.0.0.1", self.profile, self.binary, timeout) RemoteWebDriver.__init__(self, command_executor=executor, desired_capabilities=capabilities, keep_alive=True) self._is_remote = False
async def cache_clear(): rmtree('./src/cache') Path("./src/cache").mkdir(parents=True, exist_ok=True) try: bot.webdriver.quit() except: pass opts = FirefoxOptions() opts.headless = True service = Service() bot.webdriver = webdriver.Firefox(options=opts, service=service) bot.webdriver.get('https://google.com') bot.webdriver.get_screenshot_as_png() return
def before_all(context): context.server = simple_server.WSGIServer( ("", 7777), WSGIRequestHandler) #create WSGIServer instance context.server.set_app( app) #set the app that will be called on getting requests context.appthread = threading.Thread( target=context.server.serve_forever ) #create thread to call the function server_forever() context.appthread.start() #start the thread context.service = Service( "./applications/geckodriver") #include the geckodriver context.browser = webdriver.Firefox( service=context.service) #initiate the firefox browser object context.browser.set_page_load_timeout( 5000) #set the time out of the browser
def main(): profile = popen('ls ~/.mozilla/firefox/ | grep "default"').read().split( "\n")[1] user = popen("whoami").read()[0:-1] options = Options() options.set_preference('profile', "/home/" + user + "/.mozilla/firefox/" + profile) driver = webdriver.Firefox(service=Service(log_path='/dev/null'), options=options) driver.maximize_window() driver.get('https://animeschedule.net/seasons/summer-2022') data = driver.find_element(By.CLASS_NAME, 'shows-container').find_elements( By.CLASS_NAME, 'anime-tile') shows = {} for show in data: title = show.find_element( By.CLASS_NAME, 'anime-tile-title').get_attribute('innerText') print(title) cover = show.find_element(By.CLASS_NAME, 'anime-tile-thumbnail').get_attribute('src') # while "placeholder" in cover: # cover = show.find_element(By.CLASS_NAME, 'show-poster').get_attribute('src') try: studio = show.find_element( By.CLASS_NAME, 'anime-tile-stu').get_attribute('innerText') except NoSuchElementException: studio = "N/A" source = show.find_element(By.CLASS_NAME, 'anime-tile-sou').get_attribute('innerText') genres = [ ele.get_attribute('innerText') for ele in show.find_elements(By.CLASS_NAME, 'anime-tile-genre') ] content = { 'cover': cover, 'Studio': studio, 'Source': source, 'Genres': genres } shows.update({title: content}) driver.quit() shows = dict(sorted(shows.items())) with open('indent.json', 'w', encoding="utf-8") as file: dump(shows, file, indent=2) with open('shows.json', 'w', encoding="utf-8") as file: dump(shows, file, separators=(',', ':'))
def job(): try: service = Service(PATH) options = FirefoxOptions() options.headless = True profile = "./Profile" options.profile = profile driver = webdriver.Firefox(options=options, service=service) driver.get(f"https://zoom.us/j/{ID}") watch = time.strftime("%H:%M - %d/%m/%y") print(f"{c_white}[{watch}] {c_green}Joined Zoom Room {ID}{c_white}\n") file = open(r"./logger.txt", "a") file.write(f"[{watch}] Joined Zoom Room {ID}\n") time.sleep(2) driver.quit() except: watch = time.strftime("%H:%M - %d/%m/%y") print(f"{c_white}[{watch}] {c_red}Something went wrong!{c_white}\n") file = open(r"./logger.txt", "a") file.write(f"[{watch}] Something went wrong!\n")
def driver(tmp_path, binary_paths): firefox, geckodriver = binary_paths log = tmp_path / "geckodriver.log" options = Options() options.headless = True options.binary = FirefoxBinary(str(firefox)) service = Service(executable_path=str(geckodriver), service_args=["--log", "trace"], log_path=str(log)) driver = webdriver.Firefox(options=options, service=service) yield driver driver.quit() print("BEGIN geckodriver.log\n\n" f"""{log.read_text(encoding="utf-8")}""" "\n\nEND geckdriver.log")
def get_website_page_cards(flags, offset=0): """Return a list of id in a page from the website""" url = f'https://db.ygoprodeck.com/search/?{flags}&offset={offset}&view=List' browser = webdriver.Firefox(service=Service( executable_path='geckodriver.exe')) browser.get(url) time.sleep(6) soup = BeautifulSoup(browser.page_source, 'html.parser') card_list = [] for card in soup.find("div", {"id": "card-list"}).find_all('a'): card_img = card.find("div", { "id": "card-container" }).find("div", {"id": "card-img"}) try: card_id = dict(card_img.find("img").attrs)['data-src'].split( '/')[-1].split('.')[0] card_list.append(card_id) except KeyError: pass browser.close() return card_list
def __init__(self, driver): if driver is None: profile = ( popen('ls ~/.mozilla/firefox/ | grep "default"').read().split("\n")[1] ) options = Options() options.set_preference("profile", "$HOME/.mozilla/firefox/" + profile) self.driver = webdriver.Firefox( service=Service(log_path="/dev/null"), options=options ) else: self.driver = driver self.driver.maximize_window() with open("shows.json", encoding="utf-8") as file: self.shows: dict[str, ShowType] = load(file) with open("keys.json", encoding="utf-8") as file: self.keys: dict[str, str] = load(file) self.static: dict[str, ShowType] = {} self.changes: dict[str, ShowType] = {} self.new: dict[str, ShowType] = {}
def setUpClass(self): self.WEBSITE_URL = "http://127.0.0.1:8080/" try: firefox_binary = check_output(['which', 'firefox-esr']).decode().strip() except: try: firefox_binary = check_output(["which", "firefox"]).decode().strip() except: print("Could not find any firefox installed") sys.exit(1) # set up firefox service and options ops = options() ops.headless = True ops.binary_location = firefox_binary serv = Service(os.getcwd() + "/../geckodriver") self.driver = webdriver.Firefox(service=serv, options=ops)
def main(url): print("[+] Creating instance of Firefox..") DRIVER_PATH = '/usr/local/bin/geckodriver' BINARY_PATH = '/usr/bin/firefox-esr' HTB_URL = 'http://' + url ops = Options() ops.add_argument('--headless') ops.binary_location = BINARY_PATH serv = Service(DRIVER_PATH) browser = webdriver.Firefox(service=serv, options=ops) print('[+] Fetching ' + HTB_URL) browser.get(HTB_URL) string_to_be_encode = browser.find_element_by_css_selector('body h3').text md5encoded = getMD5(string_to_be_encode) print("[+] MD5 Encoded String: " + md5encoded) print("[+] Send MD5 string to input and submit") inputhtml = browser.find_element_by_css_selector('input[name="hash"]') inputhtml.send_keys(md5encoded) submithtml = browser.find_element_by_css_selector('input[type="submit"]') submithtml.click() flag = browser.find_element_by_css_selector('body p').text print("[+] Found flag: " + flag)
def setUpchrome(self): """Product name: unavailable Product version: unavailable Running Chrome headless with sendkeys requires a window size""" options = webdriver.ChromeOptions() options.add_argument("window-size=1920x1080") options.add_argument("headless") try: if self.running_platform == "Darwin": # If it's a mac, then use the old API code regardless of Selenium version handler = webdriver.Chrome(options=options, executable_path=self.handler_path + 'chromedriver') elif self.running_platform == "Windows" and WebPage.selenium_ver == "4": # If it's Windows, then check selenium version service = Service( Path(self.handler_path + 'chromedriver.exe' )) # Specify the custom path new for Selenium 4 handler = webdriver.Chrome(options=options, service=service) elif self.running_platform == "Windows": print("executable_path is", Path(self.handler_path + 'chromedriver.exe')) handler = webdriver.Chrome( options=options, executable_path=Path(self.handler_path + 'chromedriver.exe')) else: # In case it's Unix handler = webdriver.Chrome( options=options, executable_path=Path(self.handler_path + 'chromedriver')) logging.info( f"{datetime.now(tz=None)} Info Chrome browser handler found") except (WebDriverException): logging.info( f"{datetime.now(tz=None)} Warning Chrome browser handler not found or failed to launch." ) handler = None return handler
def cookie_op(self): ans = input("\nAdd a cookie with the request? (Note: This may be necessary for web sites that \n" "require authentication; Otherwise, skip this option) [y/n]: ") if ans.lower() == "n": self.cookies = None elif ans.lower() == "y": print(yellow + '\n[+] Returning cookies for ' + self.url + ': \n') try: service = Service('/usr/local/bin/geckodriver') # Fixes the deprecation warning. options = FirefoxOptions() options.add_argument("--headless") driver = webdriver.Firefox(service=service) except (IOError, OSError) as e: print("\n" + e) exit() except WebDriverException as e: print("\n" + str(e)) print("Ensure that 'geckodriver' is in the /usr/local/bin directory") exit() driver.get(self.url) print(driver.get_cookies()) print(green + '\nEx: cookiename:cookievalue:cookiepath') self.cookies = input("Enter a cookie string: ") if len(self.cookies) == 0: print("A cookie was not entered. Try again.") self.cookie_op() else: parse_cookies = self.cookies.split(':') self.cookies = { 'name': parse_cookies[0], 'value': parse_cookies[1], 'path': parse_cookies[2] } else: self.cookie_op()
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC # Add path of the geckodriver to PATH. os.environ['PATH'] += os.pathsep + 'bin/' # Update Firefox and geckodriver. os.system('check_for_updates.sh') # Set option for headless. options = Options() options.binary_location = 'bin/firefox/firefox-bin' options.add_argument('--private-window') #options.add_argument('-headless') service = Service('bin/geckodriver') # Create a new Firefox session. driver = webdriver.Firefox(service=service, options=options) # Uncomment the next line to maximize the browser window. #driver.maximize_window() # Give the browser time to load before each command is executed. driver.implicitly_wait(10) # Navigate to the application home page. driver.get('https://orteil.dashnet.org/cookieclicker/') # Wait 2 seconds for the page to finish loading. time.sleep(2)