def _start_chrome(self, user_data_dir): extra_opts = [ '--remote-debugging-port={}'.format(self._debugging_port), '--user-data-dir={}'.format(user_data_dir), '--no-sandbox', '--headless', ] command = [self._chrome_executable] + CHROME_OPTIONS + extra_opts self._p = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) self.browser = pychrome.Browser( url='http://127.0.0.1:{}'.format(self._debugging_port)) # Wait until Chrome is ready max_tries = 100 while max_tries > 0: try: self.browser.version() break except ConnectionError: time.sleep(0.1) max_tries -= 1 else: raise ChromeBrowserStartupError('Could not connect to Chrome')
def __init__ (self, browser, url, logger): self.requests = {} self.browser = pychrome.Browser (url=browser) self.url = url self.logger = logger.bind (context=type (self).__name__, url=url) self.queue = deque () self.notify = Event ()
def test_invalid_params(): browser = pychrome.Browser() tab = browser.new_tab() try: tab.Page.navigate() assert False, "never get here" except pychrome.CallMethodException: pass try: tab.Page.navigate("http://www.fatezero.org") assert False, "never get here" except pychrome.CallMethodException: pass try: tab.Page.navigate(invalid_params="http://www.fatezero.org") assert False, "never get here" except pychrome.CallMethodException: pass try: tab.Page.navigate(url="http://www.fatezero.org", invalid_params=123) except pychrome.CallMethodException: assert False, "never get here"
def __init__(self): self.hostname = "one.one.one.one" # self.loop_connected() self.options = webdriver.ChromeOptions() self.user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.22 Safari/537.36' self.port_number = self.find_free_port() self.port_url = "--remote-debugging-port=" + str(self.port_number) self.options.add_argument(f'user-agent={self.user_agent}') self.options.add_argument(self.port_url) self.options.add_argument("--disable-renderer-backgrounding") self.options.add_argument("--disable-dev-shm-usage") self.options.add_argument("--disable-gpu") # self.options.add_argument("--disable-features=NetworkService") self.options.add_argument("--disable-features=VizDisplayCompositor") self.options.add_argument("--headless") self.options.add_argument("--no-sandbox") self.driver = webdriver.Chrome(ChromeDriverManager().install(), chrome_options=self.options) self.url = "http://localhost:" + str(self.port_number) self.dev_tools = pychrome.Browser(url=self.url) self.tab = self.dev_tools.list_tab()[0] self.tab.start() # self.driver.get("https://www.google.co.in") # remove once done # self.driver.get("https://www.google.com") self.driver.get( "https://www.google.com/search?q=a&hl=en") #ENglish added
def list_fetcher(): # def pychrome(): # create a browser instance listt = [] browser = pychrome.Browser(url="http://127.0.0.1:9222") # create a tab tab = browser.new_tab() # register callback if you want def request_will_be_sent(**kwargs): # print("loading: %s" % kwargs.get('request').get('url')) listt.append(kwargs.get('request').get('url')) tab.Network.requestWillBeSent = request_will_be_sent # start the tab tab.start() # call method tab.Network.enable() # call method with timeout tab.Page.navigate(url=url) search_query = "---sn-cnoa-w5pe.googlevideo.com/videoplayback?" count = 0 while True: if len(listt) > 100: return listt
def test_get_event_listener(): browser = pychrome.Browser() tab = browser.new_tab() def request_will_be_sent(**kwargs): tab.stop() tab.start() tab.Network.requestWillBeSent = request_will_be_sent tab.Network.enable() try: tab.Page.navigate(url="chrome://newtab/") except pychrome.UserAbortException: pass if not tab.wait(timeout=5): assert False, "never get here" assert tab.Network.requestWillBeSent == request_will_be_sent tab.Network.requestWillBeSent = None assert not tab.get_listener("Network.requestWillBeSent") # notice this assert tab.Network.requestWillBeSent != tab.get_listener( "Network.requestWillBeSent") tab.stop()
def main(): browser = pychrome.Browser() close_all_tabs(browser) tabs = [] for i in range(4): tabs.append(browser.new_tab()) for i, tab in enumerate(tabs): eh = EventHandler(browser, tab) tab.Network.requestIntercepted = eh.request_intercepted tab.Page.frameStartedLoading = eh.frame_started_loading tab.Page.frameStoppedLoading = eh.frame_stopped_loading tab.start() tab.Page.stopLoading() tab.Page.enable() tab.Network.setRequestInterceptionEnabled(enabled=True) tab.Page.navigate(url="http://httpbin.org/post") for tab in tabs: tab.wait(60) tab.stop() browser.close_tab(tab) print('Done')
def start(self): if not self.chromium_executable.exists(): logger.info("Downloading Chromium browser") self.chromium_downloader.download_chromium() logger.info("Chrome Executable: {}".format(self.chromium_executable)) self.proc = subprocess.Popen([ self.chromium_executable, '--remote-debugging-port={}'.format( self.CDP_PORT), '--disable-notifications', '--disable-infobars', '--disable-breakpad', *self.flags ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) tries = 5 res = None while tries > 0: try: res = requests.get(self.cdp_url() + '/json/version') break except requests.RequestException: time.sleep(5) tries -= 1 logger.info("Waiting for browser to respond on port 9222") if res and res.status_code != 200: raise Exception("Bad status code from Chrome: " "{}".format(res.status_code)) self.browser_ws_url = res.json()['webSocketDebuggerUrl'] self.browser = pychrome.Browser(url=self.cdp_url()) self.tab_zero = self.browser.list_tab()[0] logger.info("Chromium is fired up and ready to go!")
def main(): browser = pychrome.Browser() close_all_tabs(browser) tabs = [] for i in range(len(urls)): tabs.append(browser.new_tab()) for i, tab in enumerate(tabs): eh = EventHandler(browser, tab) tab.Page.frameStartedLoading = eh.frame_started_loading tab.Page.frameStoppedLoading = eh.frame_stopped_loading tab.start() tab.Page.stopLoading() tab.Page.enable() tab.Page.navigate(url=urls[i]) for tab in tabs: tab.wait(60) tab.stop() browser.close_tab(tab) print('Done')
def __init__(self, headless: bool = False, width: int = 1920, height: int = 1080) -> None: chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--window-size=%s" % ("%d,%d" % (width, height))) if headless: chrome_options.add_experimental_option( "excludeSwitches", ["ignore-certificate-errors"]) chrome_options.add_argument( '--disable-gpu') # necessary for headless chrome_options.add_argument('--headless') chrome_options.add_argument("--hide-scrollbars") logpath = os.path.dirname( os.path.realpath(__file__)) + "/chromedriver.log" driver = webdriver.Chrome( chrome_options=chrome_options, service_args=["--verbose", "--log-path=%s" % logpath]) with open(logpath) as fp: log = fp.read() devtools_port = int( re.search(r"--remote-debugging-port=(\d+)", log).groups()[0]) # type: ignore os.remove(logpath) devtools = pychrome.Browser(url="http://127.0.0.1:%d" % devtools_port) devtools.list_tab()[0].start() self.headless = headless self.driver = driver self.devtools = devtools self.chrome_options = chrome_options self.enableTab(0)
def list_fetcher(): # def pychrome(): # create a browser instance listt = [] browser = pychrome.Browser(url="http://127.0.0.1:9222") # create a tab tab = browser.new_tab() # register callback if you want def request_will_be_sent(**kwargs): # print("loading: %s" % kwargs.get('request').get('url')) listt.append(kwargs.get('request').get('url')) tab.Network.requestWillBeSent = request_will_be_sent # start the tab tab.start() # call method tab.Network.enable() # call method with timeout tab.Page.navigate(url=url) # wait for loading tab.wait(5) browser.close_tab(tab) return listt
def start_browser(self): # Start chromedriver options = webdriver.ChromeOptions() options.add_argument('--disable-background-networking=false') options.add_argument('--no-sandbox') retry_count = 0 while True: try: self.browser = webdriver.Chrome( chrome_options=options, service_args=[ '--verbose', '--log-path=./logs/{id}.log'.format(id=self.id) ]) break except ConnectionResetError as e: retry_count += 1 if retry_count >= 10: raise e self.browser.set_page_load_timeout(self.TIMEOUT) self.wait = WebDriverWait(self.browser, self.TIMEOUT) # Get debug url url = None with open('./logs/{id}.log'.format(id=self.id), 'r') as log: for line in log: if 'DevTools request: http://localhost' in line: url = line[line.index('http'):].replace( '/json/version', '').strip() break if not url: raise Exception('Invalid protocol url.') # Start pychrome self.chrome = pychrome.Browser(url=url)
def html_to_pdf(self, html, delay=5): """ :param html: html document as a bytestring :param delay: time to wait for javascript loading in seconds :return: path to a temporary file """ split = urlparse(settings.CHROME_URL) ipaddr = gethostbyname(split.hostname) # related to https://github.com/GoogleChrome/puppeteer/issues/2242 url = settings.CHROME_URL.replace(split.hostname, ipaddr) browser = pychrome.Browser(url=url) encoded_html = base64.b64encode(html) data_url = "data:text/html;base64,{}".format( encoded_html.decode('utf-8')) tab = browser.new_tab(data_url) tab.start() tab.wait(delay) try: data = tab.Page.printToPDF() data = base64.b64decode(data['data']) return ContentFile(data) finally: tab.stop() browser.close_tab(tab)
def test_status(): browser = pychrome.Browser() tab = browser.new_tab() assert tab.status() == pychrome.Tab.status_initial def request_will_be_sent(**kwargs): tab.stop() tab.Network.requestWillBeSent = request_will_be_sent assert tab.status() == pychrome.Tab.status_initial tab.Network.enable() assert tab.status() == pychrome.Tab.status_started try: tab.Page.navigate(url="chrome://newtab/") except pychrome.UserAbortException: pass if not tab.wait(timeout=5): assert False, "never get here" assert tab.status() == pychrome.Tab.status_stopped
def browser(): # A bug in Travis-CI's testing environment requires that chrome requires # sudo access to set up the sandbox. Hence we turn it off. # A regular environment does not need these service args service_args = ['--no-sandbox', '--disable-gpu'] with pychrome.Browser(service_args=service_args) as browser: yield browser
def test_browser_activate_tab(): browser = pychrome.Browser() tabs = [] for i in range(10): tabs.append(browser.new_tab()) for tab in tabs: browser.activate_tab(tab)
def close(host, port, secure, id): """close a target/tab by id""" url = "%s://%s:%s" % ("https" if secure else "http", host, port) try: browser = pychrome.Browser(url) click.echo(browser.close_tab(id)) except Exception as e: click.echo(e)
def list(host, port, secure): """list all the available targets/tabs""" url = "%s://%s:%s" % ("https" if secure else "http", host, port) try: browser = pychrome.Browser(url) click.echo(json.dumps(browser.list_tab(), cls=JSONTabEncoder, indent=4)) except Exception as e: click.echo(e)
def __init__(self): # google-chrome --headless --disable-gpu --remote-debugging-port=9222 # os.system("F:\\software\\Google\\Chrome\\Application\\chrome.exe --headless --disable-gpu --remote-debugging-port=9222") # cc = os.popen("F:\\software\\Google\\Chrome\\Application\\chrome.exe --headless --disable-gpu --remote-debugging-port=9222") # create a browser instance self.browser = pychrome.Browser(url="http://127.0.0.1:9222") self.url = "http://quote.eastmoney.com/concept/" self.result = []
def test_set_wrong_listener(): browser = pychrome.Browser() tab = browser.new_tab() try: tab.Network.requestWillBeSent = "test" assert False, "never get here" except pychrome.RuntimeException: pass
def test_browser_tabs_map(): browser = pychrome.Browser() tab = browser.new_tab() assert tab in browser.list_tab() assert tab in browser.list_tab() browser.close_tab(tab) assert tab not in browser.list_tab()
def version(host, port, secure): """show the browser version""" url = "%s://%s:%s" % ("https" if secure else "http", host, port) try: browser = pychrome.Browser(url) click.echo(json.dumps(browser.version(), indent=4)) except Exception as e: click.echo(e)
def test_invalid_method(): browser = pychrome.Browser() tab = browser.new_tab() try: tab.Page.NotExistMethod() assert False, "never get here" except pychrome.CallMethodException: pass
def new(host, port, secure, url="about:blank"): """create a new target/tab""" _url = "%s://%s:%s" % ("https" if secure else "http", host, port) try: browser = pychrome.Browser(_url) click.echo( json.dumps(browser.new_tab(url), cls=JSONTabEncoder, indent=4)) except Exception as e: click.echo(e)
def __init__(self, *args, **kwargs): # create a browser instance self.browser = pychrome.Browser(url="http://127.0.0.1:9221") # create a tab self.tab = self.browser.new_tab() #self.tab.Network.requestWillBeSent = self.request_will_be_sent # start the tab self.tab.start() # call method self.tab.Network.enable()
def __init__(self, log_id, port_number, agent_name, browser_pid): self.agent_name = agent_name self.logger = logging.getLogger(log_id) self.port = port_number print "Devtools listening on this port: ", self.port self.browser = pychrome.Browser(url="http://127.0.0.1:%s" % (self.port,)) # Contains EventHandler objects for each tab; tab_id => EventHandler obj self.eh = {} self.download_path = os.path.join(MAIN_LOG_PATH, RAW_DOWNLOADS_DIR) self.last_window_open_url = None self.browser_pid = browser_pid
def __init__(self, url="http://127.0.0.1:9222", _request_will_be_sent=None, _web_socket_created=None): if _web_socket_created is None: _web_socket_created = self.web_socket_created if _request_will_be_sent is None: _request_will_be_sent = self.request_will_be_sent self.browser = pychrome.Browser(url=url) self.tab = self.browser.new_tab() self.tab.set_listener("Network.requestWillBeSent", _request_will_be_sent) self.tab.set_listener("Network.webSocketFrameReceived", _web_socket_created)
def test_normal_callmethod(): browser = pychrome.Browser() tab = browser.new_tab() result = tab.Page.navigate(url="http://www.fatezero.org") assert result['frameId'] time.sleep(1) result = tab.Runtime.evaluate(expression="document.domain") assert result['result']['type'] == 'string' assert result['result']['value'] == 'www.fatezero.org'
def __init__(self, index, multiThreaded=False): self.number = index self.browser = pychrome.Browser() tabs = self.browser.list_tab() # if not tabs: self.tab = self.browser.new_tab() # else: # self.tab = tabs[0] # self.tab.debug = True self.tab.start() self.tab.Page.enable() self.tab.Page.navigate( url="http://www.foddy.net/Athletics.html?webgl=true") if (multiThreaded): self.tab.call_method("Page.startScreencast", format="png", quality=25, everyNthFrame=5, maxWidth=640, maxHeight=400) self.tab.Page.screencastFrame = self.processStreamFrame # self.gameIntroTemplate = cv2.resize(cv2.imread('intro.png', 0), (0,0), fx=0.68, fy=0.68) if multiThreaded: self.gameIntroTemplate = cv2.imread('intro.png', 0) self.gameLostTemplate = cv2.imread('lost.png', 0) else: self.gameIntroTemplate = cv2.imread('intro-small.png', 0) self.gameLostTemplate = cv2.imread('lost-small.png', 0) self.grayImage = None self.image = None self.scoreImage = None self.imageIsSimilar = False self.scoreIsSimilar = False self.previousKey = None # cv2.namedWindow(str(self.number)) # while True: while self.image is None: if not multiThreaded: self.takeScreenshot() time.sleep(1)
def test_callback_exception(): browser = pychrome.Browser() tab = browser.new_tab() def request_will_be_sent(**kwargs): raise Exception("will not stop") tab.Network.requestWillBeSent = request_will_be_sent tab.Network.enable() tab.Page.navigate(url="chrome://newtab/") if tab.wait(timeout=3): assert False, "never get here"