class DriverBase(object): def __init__(self, config, profile): self.config = config self.options = Options() #self.options.add_argument("lang=en-US") self.options.add_argument("mute-audio") version = self.choose_version(self.config['opera_path']) self.options.binary_location = os.path.join(self.config['opera_path'], version, 'opera.exe') self.options.add_argument('user-data-dir=' + profile) self.d = None # instance of the webdriver def _elem_to_str(self, e): return e.get_attribute('outerHTML') def _print_all_elems(self, e): for x in e.find_elements_by_css_selector("*"): print(self._elem_to_str(x)) # choose the latest browser version in the folder def choose_version(self, folder): return sorted([ f.name for f in os.scandir(folder) if f.is_dir() and re.match(r'[\d.]+', f.name) ])[-1] def open_browser(self): self.d = webdriver.Opera(options=self.options)
def driver(browser): if browser == "firefox": firefox_options = FirefoxOptions() firefox_options.add_argument("headless") driver = webdriver.Firefox(options=firefox_options) driver.maximize_window() elif browser == "opera": opera_options = OperaOptions() opera_options.add_argument("headless") driver = webdriver.Opera(options=opera_options) driver.maximize_window() else: chrome_options = ChromeOptions() chrome_options.headless = False chrome_options.add_argument("headless") driver = webdriver.Chrome( executable_path='homework_Selenium/driver/chromedriver', options=chrome_options) driver.maximize_window() constants.DRIVER = driver yield if sys.exc_info(): constants.DRIVER.quit()
def opera_second(): options = Options() profile = 'C:/Users/user/Desktop/idiq/bin/second/Opera Software/Opera Stable' options.add_argument('user-data-dir=' + profile) options.add_argument('private') options.add_argument("--kiosk-printing") options.add_argument( "download.default_directory=C:/Users/{username}/Desktop/") global driver driver = webdriver.Opera(options=options) driver.get('https://www.identityiq.com/login.aspx') time.sleep(2) IDIQ.logon(acc[0], acc[1])
def driver_factory(browser_name: str, is_headless: bool = False, webdriver_logging: bool = False) -> webdriver: """ Return webdriver object for a specified browser name """ driver = None if browser_name == "firefox": firefox_options = webdriver.FirefoxOptions() if is_headless: firefox_options.add_argument("--headless") if webdriver_logging: driver = EventFiringWebDriver( driver=webdriver.Firefox(options=firefox_options), event_listener=DriverEventListener( log_filename=f"{browser_name}.log")) else: driver = webdriver.Firefox(options=firefox_options) elif browser_name == "chrome": chrome_options = webdriver.ChromeOptions() if is_headless: chrome_options.add_argument("--headless") capabilities = DesiredCapabilities.CHROME.copy() capabilities['acceptSslCerts'] = True capabilities['acceptInsecureCerts'] = True capabilities['loggingPrefs'] = {'browser': 'ALL'} if webdriver_logging: driver = EventFiringWebDriver( driver=webdriver.Chrome(options=chrome_options, desired_capabilities=capabilities), event_listener=DriverEventListener( log_filename=f"{browser_name}.log")) else: driver = webdriver.Chrome(options=chrome_options, desired_capabilities=capabilities) elif browser_name == "yandex": # TODO: implement Yandex browser pass elif browser_name == "opera": opera_options = OperaOptions() opera_options.binary_location = OPERA_BROWSER_EXECUTABLE if is_headless: opera_options.add_argument("--headless") capabilities = DesiredCapabilities.OPERA.copy() capabilities['acceptSslCerts'] = True capabilities['acceptInsecureCerts'] = True if webdriver_logging: driver = EventFiringWebDriver( driver=webdriver.Opera( options=opera_options, executable_path=OPERA_WEBDRIVER_EXECUTABLE, desired_capabilities=capabilities), event_listener=DriverEventListener( log_filename=f"{browser_name}.log")) else: driver = webdriver.Opera( options=opera_options, executable_path=OPERA_WEBDRIVER_EXECUTABLE, desired_capabilities=capabilities) elif browser_name == "safari": # TODO: implement Safari browser pass else: raise NameError("Browser not supported") return driver
def setup_method(self, method): self.assert_errors = '\n' browser_name = os.environ.get('BROWSER') if browser_name == 'firefox': profile = webdriver.FirefoxProfile() profile.set_preference("devtools.console.stdout.content", True) # if hasattr(method, 'portable'): # profile.set_preference( # "general.useragent.override", # "Mozilla/5.0 (Linux; Android 8.0;" # "Pixel 2 Build/OPD3.170816.012) " # "AppleWebKit/537.36 (KHTML, like Gecko)" # "Chrome/70.0.3538.77 Mobile Safari/537.36" # ) # profile.update_preferences() firefox_options = FirefoxOptions() firefox_options.add_argument("-no-remote") firefox_options.log.level = 'trace' self.driver = webdriver.Firefox(firefox_profile=profile, options=firefox_options) elif browser_name == 'opera': opera_options = OperaOptions() opera_options.add_argument("--verbose") opera_options.add_argument("--enable-logging --v=1") opera_options.add_argument("--no-sandbox") opera_options.add_argument("--ignore-certificate-errors") opera_options.add_argument("--disable-notifications") opera_options.add_argument("--disable-gpu") opera_options.add_experimental_option('w3c', False) # if hasattr(method, 'portable'): # opera_options.add_experimental_option( # "mobileEmulation", # {'deviceName': 'Nexus 5'} # ) opera_options.binary_location = '/usr/bin/opera' self.driver = webdriver.Opera(options=opera_options) else: options = ChromeOptions() options.add_argument("--verbose") options.add_argument("--enable-logging --v=1") options.add_argument("--no-sandbox") options.add_argument("--ignore-certificate-errors") options.add_argument("--disable-notifications") options.add_argument("--disable-gpu") options.add_experimental_option('w3c', False) if hasattr(method, 'portable'): options.add_experimental_option("mobileEmulation", {'deviceName': 'Nexus 5'}) self.driver = webdriver.Chrome(chrome_options=options) self.driver.maximize_window()
c.value = data print(data) # Opening the excel file and reading the URL wb = xl.load_workbook('Web Scrapping Task..xlsx') sheet = wb['Sheet1'] row_value = sheet.cell(row=2, column=1) # print(row_value.value) url_value = row_value.value.split(":", 1)[1] # print(url_value) # Using opera driver for automation # Running in headless mode opt = Options() opt.add_argument("--headless") driver = webdriver.Opera( executable_path='C:\\Users\\hp\\Downloads\\Driver For ' 'Automation\\operadriver_win64\\operadriver_win64\\operadriver.exe') # Opening the site in full screen mode driver.maximize_window() driver.get(url_value) # Wait for Page to Load time.sleep(30) # Login to facebook driver.find_element_by_xpath( "(//*[@class='m9osqain jq4qci2q a3bd9o3v'])[1]").click() driver.find_element_by_xpath("(//*[@class='m9osqain jq4qci2q a3bd9o3v'])[1]"
def translate(src_lang, tgt_lang, list_text): global translate_file_name list_tgt = list() count = 1 search_text = "" sign = ";" #f = open("link.test.txt", "w", encoding="utf-8") loop = 1 opera_profile = "C:\\Users\\\Admin\\AppData\\Roaming\\Opera Software\\Opera Stable" options = Options() options.binary_location = r'C:\Users\\Admin\AppData\Local\Programs\Opera\74.0.3911.107\opera.exe' options.add_argument("--headless") options.add_argument('--no-sandbox') #options.add_argument('--user-data-dir=' + opera_profile) #driver = webdriver.PhantomJS(os.path.abspath(os.getcwd()+"\phantomjs")) #driver = webdriver.Opera(executable_path=os.path.abspath(os.getcwd()+"/operadriver.exe"), options = options) driver = ChromeDriver.getChromeDriver() url = "https://translate.google.com/?sl={}&tl={}&text={}&op=translate".format( src_lang, tgt_lang, "") driver.get(url) textarea = driver.find_element_by_tag_name("textarea") old_text = "" print(len(list_text)) len_of_batch = 0 lim_of_batch = 3800 text_to_translate = "" list_transed_text = list() try: for text in list_text: if (loop % 100 == 0): driver.close() driver = ChromeDriver.getChromeDriver() url = "https://translate.google.com/?sl={}&tl={}&text={}&op=translate".format( src_lang, tgt_lang, "") driver.get(url) textarea = driver.find_element_by_tag_name("textarea") loop = 1 textarea = driver.find_element_by_tag_name("textarea") if (len_of_batch > lim_of_batch): last_index = len(text_to_translate) - 1 if (text_to_translate[last_index] == "\n"): text_to_translate = text_to_translate[0:last_index - 1] textarea.send_keys(text_to_translate) list_transed_text = getTranslateContent(driver, list_tgt) textarea.clear() list_tgt = list() len_of_batch = 0 text_to_translate = "" loop = loop + 1 time.sleep(0.5) else: text_len = len(text) if (text_len + len_of_batch > 4000): last_index = len(text_to_translate) - 1 if (text_to_translate[last_index] == "\n"): text_to_translate = text_to_translate[0:last_index - 1] textarea.send_keys(text_to_translate) list_transed_text = getTranslateContent(driver, list_tgt) textarea.clear() len_of_batch = 0 list_tgt = list() loop = loop + 1 text_to_translate = "" time.sleep(0.5) len_of_batch = len_of_batch + text_len text = bytes(text, "utf-8").decode('utf-8', 'ignore') text = text.replace("\r", "") not_bmp = x = re.search(u'[\U00010000-\U0010ffff]', text) if not_bmp: continue text_to_translate = text_to_translate + text + " \n" list_tgt.append(text) if list_transed_text: saveFile(translate_file_name, list_transed_text) list_transed_text = list() except Exception as e: print(e) traceback.print_exc() time.sleep(3) #pdb.set_trace() pass #f.close() if (len_of_batch > 0): textarea.send_keys(text_to_translate) list_transed_text = getTranslateContent(driver, list_tgt) if list_transed_text: if (len(list_transed_text) > 0): saveFile(translate_file_name, list_transed_text) driver.close()
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2866.71 Safari/537.36", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2919.83 Safari/537.36", "Mozilla/5.0 (X11; Ubuntu; Linux i686 on x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2820.59 Safari/537.36" ] headers = { 'User-Agent': random.choice(uastrings), 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip', 'DNT': '1' } options = Options() options.headless = False options.add_argument("--incognito") driver = webdriver.Opera(options=options, executable_path=DRIVER_PATH) url_temp = "https://tuoitre.vn/" + 'the-gioi' + "/trang-" + str(100 - 1) + ".htm" driver.get(url_temp) driver.implicitly_wait(30) click = 1 try: while click < 1000: try: driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 10).until( EC.visibility_of_element_located((By.CLASS_NAME,"btn-readmore")))) driver.execute_script("arguments[0].click();", WebDriverWait(driver, 10).until(EC.element_to_be_clickable(
from selenium import webdriver from selenium.webdriver.opera.options import Options import time import requests import os import json # required to do if opera is not installed in default loacation options = Options() options.binary_location = "A:\\Opera\\launcher.exe" options.add_argument("--remote-debugging-port=9222") # driver initiliazation driver_path = webdriver.Opera( executable_path= "C:\\Users\\nouamane\\Downloads\\Programms\\operadriver.exe", options=options) def play_video(): # global youtube watch url watch_on_yotube = 'http://www.youtube.com/watch?v=' api_key = os.environ.get( 'TUBE_KEY') # get it from google developer api key channel_id = 'UCwO_xoYm2vjhu4kZSxFO5mA' # your favorite youtuber id channel base_url = 'https://www.googleapis.com/youtube/v3/search?' # optional parameters to be passed to the url
def driver(request): logger = logging.getLogger('BrowserLogger') browser = request.config.getoption("--browser") version = request.config.getoption("--bversion") executor = request.config.getoption("--executor") vnc = request.config.getoption("--vnc") logs = request.config.getoption("--logs") videos = request.config.getoption("--videos") executor_url = f"http://{executor}:4444/wd/hub" mobile = request.config.getoption("--mobile") is_remote = request.config.getoption("--is_remote") caps = { "browserName": browser, "browserVersion": version, "screenResolution": "1280x720", "name": "Tester", "selenoid:options": { "enableVNC": vnc, "enableVideo": videos, "enableLog": logs, }, 'acceptSslCerts': True, 'acceptInsecureCerts': True, 'timeZone': 'Europe/Moscow', 'goog:chromeOptions': { 'args': [] } } mobile_emulation = {"deviceName": "iPhone 5/SE"} if is_remote: if browser == "chrome" and mobile: caps["goog:chromeOptions"]["mobileEmulation"] = mobile_emulation driver = wd.Remote( command_executor=executor_url, desired_capabilities=caps ) else: if browser == "firefox": firefox_options = FirefoxOptions() firefox_options.headless = True driver = wd.Firefox(options=firefox_options) elif browser == "opera": opera_options = OperaOptions() opera_options.add_argument("headless") driver = wd.Opera(options=opera_options) elif browser == "chrome": chrome_options = ChromeOptions() # chrome_options.add_argument("headless") if mobile: chrome_options.add_experimental_option("mobileEmulation", mobile_emulation) driver = wd.Chrome(options=chrome_options) else: raise ValueError(f"{browser} browser not supported") if not mobile: driver.maximize_window() logger.info(f"Browser {browser} started.") driver = EventFiringWebDriver(driver, MyListener()) # Attach browser data allure.attach( name=driver.session_id, body=json.dumps(driver.desired_capabilities), attachment_type=allure.attachment_type.JSON) # Add environment info to allure-report with open("allure-report/environment.xml", "w+") as file: file.write(f"""<environment> <parameter> <key>Browser</key> <value>{browser}</value> </parameter> <parameter> <key>Browser.Version</key> <value>{version}</value> </parameter> <parameter> <key>Executor</key> <value>{executor}</value> </parameter> </environment> """) def teardown(): driver.quit() logger.info(f"Browser {browser} closed") request.addfinalizer(teardown) return driver
self.items.delete_many({}) def insert(self, entity): self.items.insert_one(entity) def head(self, num): objects = self.items.find().limit(num) for item in objects: print(item) def contains(self, item): return self.items.find(item).count() if __name__ == '__main__': opera_options = Options() opera_options.add_argument('--start-maximized') driver = webdriver.Opera(options=opera_options) mongo = Mongo(blank=True) try: links = get_pages(driver) for link in links: data = get_data(link) if not mongo.contains(data): mongo.insert(data) driver.quit() except Exception as e: print(e) driver.quit()
driver.get('https://whatismyipaddress.com') sleep(10) driver.quit() from selenium import webdriver from time import sleep # The profile where I enabled the VPN previously using the GUI. options = webdriver.ChromeOptions() options.add_argument("--user-data-dir=chrome-data/") driver = webdriver.Opera(executable_path='operadriver.exe', options=options) driver.get('https://whatismyipaddress.com') sleep(10) driver.quit() from selenium.webdriver.chrome.options import Options from selenium import webdriver options = Options() options.add_argument("--user-data-dir=chrome-data/") pluginfile = 'HMA-VPN-Proxy-Unblocker.zip' options.add_extension(pluginfile) driver = webdriver.Chrome(executable_path='chromedriver.exe', chrome_options=options) driver.get( 'https://my.hidemyass.com/?utm_medium=prg_link&utm_source=extension_chrome&utm_campaign=hma_mvp#login' )
def dragonfly(cls, target_url='', params={}, headers={}, cookies={}, method="get", return_params=False, for_cookies=False, proxy_pwd="", proxy_user="", proxy_host="dyn.horocn.com", proxy_port="50000"): resp = '' if 1 > len(target_url) or 1 > len(proxy_user) or 1 > len(proxy_pwd): return (-1, f"wrong parameters") if not isinstance(headers, dict) or 1 > len(headers): headers = { "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1", "Referer": "http://c.easygo.qq.com/eg_toc/map.html?origin=csfw", } proxy_meta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % { "host": proxy_host, "port": proxy_port, "user": proxy_user, "pass": proxy_pwd, } proxies = { "httpProxy": proxy_meta, "noProxy": None, "proxyType": "MANUAL", "class": "org.openqa.selenium.Proxy", "autodetect": False, } try: options = Options() options.add_argument('accept="application/json"') options.add_argument( 'referer="http://c.easygo.qq.com/eg_toc/map.html?origin="') options.add_argument( 'user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"' ) options.add_argument('lang=zh_CN.UTF-8') desired_capabilities = options.to_capabilities() desired_capabilities['proxy'] = proxies # options.add_argument('--headless') # 在广州分公司将无头配置项注销掉,直接在windows下运行 # https://blog.csdn.net/vinson0526/article/details/51850929 # driver = webdriver.Chrome(desired_capabilities = desired_capabilities, options=options, executable_path="chromedriver.exe") # 需要使用相同目录下的这个版本的chromedriver.exe;否则报版本错误 # https://get.geo.opera.com/pub/opera/desktop/58.0.3135.47/win/ # https://github.com/operasoftware/operachromiumdriver/releases # this one is only for Windows NT, cannot for Win10 Home Edition # executable_path="C:\\Entrobus\\projects\\crawl\\drivers\\operadriver.exe" driver = webdriver.Opera(desired_capabilities=desired_capabilities, options=options, executable_path="chromedriver.exe") driver.implicitly_wait(30) driver.get( "http://c.easygo.qq.com/eg_toc/map.html?origin=csfw&cityid=110000" ) # qq_num = "3382624374" # qq_passwd = "co07s9a4w6e" qq_num = "1934267750" qq_passwd = "yu101472" driver.find_element_by_id("u").send_keys(qq_num) driver.find_element_by_id("p").send_keys(qq_passwd) driver.find_element_by_id("go").click() #检查是否存在验证码 if "手机统一登录" in driver.title: # 成功登陆以后变成:宜出行 # raise CookieException print(f"the qq number need to safe verify is {qq_num}") time.sleep(35) cookies = driver.get_cookies() driver.quit() driver.close() # save cookie cookies_dict = {} for cookie in cookies: cookies_dict[cookie["name"]] = cookie["value"] cookies_string = "" for index, one in enumerate(cookies_dict): cookies_string += f"{str( one)}:{cookies_dict[one]};" cookies_string = cookies_string.strip(";") return (200, cookies_string) except Exception as ex: driver.close() return ( -2, f"Exception happens while crawling {target_url} Exception = {ex}" )