def _start_chrome(self, user_data_dir):
extra_opts = [
'--remote-debugging-port={}'.format(self._debugging_port),
'--user-data-dir={}'.format(user_data_dir),
'--no-sandbox',
'--headless',
]
command = [self._chrome_executable] + CHROME_OPTIONS + extra_opts
self._p = subprocess.Popen(command,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
self.browser = pychrome.Browser(
url='http://127.0.0.1:{}'.format(self._debugging_port))
# Wait until Chrome is ready
max_tries = 100
while max_tries > 0:
try:
self.browser.version()
break
except ConnectionError:
time.sleep(0.1)
max_tries -= 1
else:
raise ChromeBrowserStartupError('Could not connect to Chrome')
def __init__ (self, browser, url, logger):
self.requests = {}
self.browser = pychrome.Browser (url=browser)
self.url = url
self.logger = logger.bind (context=type (self).__name__, url=url)
self.queue = deque ()
self.notify = Event ()
def test_invalid_params():
browser = pychrome.Browser()
tab = browser.new_tab()
try:
tab.Page.navigate()
assert False, "never get here"
except pychrome.CallMethodException:
pass
try:
tab.Page.navigate("http://www.fatezero.org")
assert False, "never get here"
except pychrome.CallMethodException:
pass
try:
tab.Page.navigate(invalid_params="http://www.fatezero.org")
assert False, "never get here"
except pychrome.CallMethodException:
pass
try:
tab.Page.navigate(url="http://www.fatezero.org", invalid_params=123)
except pychrome.CallMethodException:
assert False, "never get here"
def __init__(self):
self.hostname = "one.one.one.one"
# self.loop_connected()
self.options = webdriver.ChromeOptions()
self.user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.22 Safari/537.36'
self.port_number = self.find_free_port()
self.port_url = "--remote-debugging-port=" + str(self.port_number)
self.options.add_argument(f'user-agent={self.user_agent}')
self.options.add_argument(self.port_url)
self.options.add_argument("--disable-renderer-backgrounding")
self.options.add_argument("--disable-dev-shm-usage")
self.options.add_argument("--disable-gpu")
# self.options.add_argument("--disable-features=NetworkService")
self.options.add_argument("--disable-features=VizDisplayCompositor")
self.options.add_argument("--headless")
self.options.add_argument("--no-sandbox")
self.driver = webdriver.Chrome(ChromeDriverManager().install(),
chrome_options=self.options)
self.url = "http://localhost:" + str(self.port_number)
self.dev_tools = pychrome.Browser(url=self.url)
self.tab = self.dev_tools.list_tab()[0]
self.tab.start()
# self.driver.get("https://www.google.co.in") # remove once done
# self.driver.get("https://www.google.com")
self.driver.get(
"https://www.google.com/search?q=a&hl=en") #ENglish added
def list_fetcher():
# def pychrome():
# create a browser instance
listt = []
browser = pychrome.Browser(url="http://127.0.0.1:9222")
# create a tab
tab = browser.new_tab()
# register callback if you want
def request_will_be_sent(**kwargs):
# print("loading: %s" % kwargs.get('request').get('url'))
listt.append(kwargs.get('request').get('url'))
tab.Network.requestWillBeSent = request_will_be_sent
# start the tab
tab.start()
# call method
tab.Network.enable()
# call method with timeout
tab.Page.navigate(url=url)
search_query = "---sn-cnoa-w5pe.googlevideo.com/videoplayback?"
count = 0
while True:
if len(listt) > 100:
return listt
def test_get_event_listener():
browser = pychrome.Browser()
tab = browser.new_tab()
def request_will_be_sent(**kwargs):
tab.stop()
tab.start()
tab.Network.requestWillBeSent = request_will_be_sent
tab.Network.enable()
try:
tab.Page.navigate(url="chrome://newtab/")
except pychrome.UserAbortException:
pass
if not tab.wait(timeout=5):
assert False, "never get here"
assert tab.Network.requestWillBeSent == request_will_be_sent
tab.Network.requestWillBeSent = None
assert not tab.get_listener("Network.requestWillBeSent")
# notice this
assert tab.Network.requestWillBeSent != tab.get_listener(
"Network.requestWillBeSent")
tab.stop()
def main():
browser = pychrome.Browser()
close_all_tabs(browser)
tabs = []
for i in range(4):
tabs.append(browser.new_tab())
for i, tab in enumerate(tabs):
eh = EventHandler(browser, tab)
tab.Network.requestIntercepted = eh.request_intercepted
tab.Page.frameStartedLoading = eh.frame_started_loading
tab.Page.frameStoppedLoading = eh.frame_stopped_loading
tab.start()
tab.Page.stopLoading()
tab.Page.enable()
tab.Network.setRequestInterceptionEnabled(enabled=True)
tab.Page.navigate(url="http://httpbin.org/post")
for tab in tabs:
tab.wait(60)
tab.stop()
browser.close_tab(tab)
print('Done')
def start(self):
if not self.chromium_executable.exists():
logger.info("Downloading Chromium browser")
self.chromium_downloader.download_chromium()
logger.info("Chrome Executable: {}".format(self.chromium_executable))
self.proc = subprocess.Popen([
self.chromium_executable, '--remote-debugging-port={}'.format(
self.CDP_PORT), '--disable-notifications',
'--disable-infobars', '--disable-breakpad', *self.flags
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
tries = 5
res = None
while tries > 0:
try:
res = requests.get(self.cdp_url() + '/json/version')
break
except requests.RequestException:
time.sleep(5)
tries -= 1
logger.info("Waiting for browser to respond on port 9222")
if res and res.status_code != 200:
raise Exception("Bad status code from Chrome: "
"{}".format(res.status_code))
self.browser_ws_url = res.json()['webSocketDebuggerUrl']
self.browser = pychrome.Browser(url=self.cdp_url())
self.tab_zero = self.browser.list_tab()[0]
logger.info("Chromium is fired up and ready to go!")
def main():
browser = pychrome.Browser()
close_all_tabs(browser)
tabs = []
for i in range(len(urls)):
tabs.append(browser.new_tab())
for i, tab in enumerate(tabs):
eh = EventHandler(browser, tab)
tab.Page.frameStartedLoading = eh.frame_started_loading
tab.Page.frameStoppedLoading = eh.frame_stopped_loading
tab.start()
tab.Page.stopLoading()
tab.Page.enable()
tab.Page.navigate(url=urls[i])
for tab in tabs:
tab.wait(60)
tab.stop()
browser.close_tab(tab)
print('Done')
def __init__(self,
headless: bool = False,
width: int = 1920,
height: int = 1080) -> None:
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--window-size=%s" % ("%d,%d" %
(width, height)))
if headless:
chrome_options.add_experimental_option(
"excludeSwitches", ["ignore-certificate-errors"])
chrome_options.add_argument(
'--disable-gpu') # necessary for headless
chrome_options.add_argument('--headless')
chrome_options.add_argument("--hide-scrollbars")
logpath = os.path.dirname(
os.path.realpath(__file__)) + "/chromedriver.log"
driver = webdriver.Chrome(
chrome_options=chrome_options,
service_args=["--verbose", "--log-path=%s" % logpath])
with open(logpath) as fp:
log = fp.read()
devtools_port = int(
re.search(r"--remote-debugging-port=(\d+)",
log).groups()[0]) # type: ignore
os.remove(logpath)
devtools = pychrome.Browser(url="http://127.0.0.1:%d" % devtools_port)
devtools.list_tab()[0].start()
self.headless = headless
self.driver = driver
self.devtools = devtools
self.chrome_options = chrome_options
self.enableTab(0)
def list_fetcher():
# def pychrome():
# create a browser instance
listt = []
browser = pychrome.Browser(url="http://127.0.0.1:9222")
# create a tab
tab = browser.new_tab()
# register callback if you want
def request_will_be_sent(**kwargs):
# print("loading: %s" % kwargs.get('request').get('url'))
listt.append(kwargs.get('request').get('url'))
tab.Network.requestWillBeSent = request_will_be_sent
# start the tab
tab.start()
# call method
tab.Network.enable()
# call method with timeout
tab.Page.navigate(url=url)
# wait for loading
tab.wait(5)
browser.close_tab(tab)
return listt
def start_browser(self):
# Start chromedriver
options = webdriver.ChromeOptions()
options.add_argument('--disable-background-networking=false')
options.add_argument('--no-sandbox')
retry_count = 0
while True:
try:
self.browser = webdriver.Chrome(
chrome_options=options,
service_args=[
'--verbose',
'--log-path=./logs/{id}.log'.format(id=self.id)
])
break
except ConnectionResetError as e:
retry_count += 1
if retry_count >= 10:
raise e
self.browser.set_page_load_timeout(self.TIMEOUT)
self.wait = WebDriverWait(self.browser, self.TIMEOUT)
# Get debug url
url = None
with open('./logs/{id}.log'.format(id=self.id), 'r') as log:
for line in log:
if 'DevTools request: http://localhost' in line:
url = line[line.index('http'):].replace(
'/json/version', '').strip()
break
if not url:
raise Exception('Invalid protocol url.')
# Start pychrome
self.chrome = pychrome.Browser(url=url)
def html_to_pdf(self, html, delay=5):
"""
:param html: html document as a bytestring
:param delay: time to wait for javascript loading in seconds
:return: path to a temporary file
"""
split = urlparse(settings.CHROME_URL)
ipaddr = gethostbyname(split.hostname)
# related to https://github.com/GoogleChrome/puppeteer/issues/2242
url = settings.CHROME_URL.replace(split.hostname, ipaddr)
browser = pychrome.Browser(url=url)
encoded_html = base64.b64encode(html)
data_url = "data:text/html;base64,{}".format(
encoded_html.decode('utf-8'))
tab = browser.new_tab(data_url)
tab.start()
tab.wait(delay)
try:
data = tab.Page.printToPDF()
data = base64.b64decode(data['data'])
return ContentFile(data)
finally:
tab.stop()
browser.close_tab(tab)
def test_status():
browser = pychrome.Browser()
tab = browser.new_tab()
assert tab.status() == pychrome.Tab.status_initial
def request_will_be_sent(**kwargs):
tab.stop()
tab.Network.requestWillBeSent = request_will_be_sent
assert tab.status() == pychrome.Tab.status_initial
tab.Network.enable()
assert tab.status() == pychrome.Tab.status_started
try:
tab.Page.navigate(url="chrome://newtab/")
except pychrome.UserAbortException:
pass
if not tab.wait(timeout=5):
assert False, "never get here"
assert tab.status() == pychrome.Tab.status_stopped
def browser():
# A bug in Travis-CI's testing environment requires that chrome requires
# sudo access to set up the sandbox. Hence we turn it off.
# A regular environment does not need these service args
service_args = ['--no-sandbox', '--disable-gpu']
with pychrome.Browser(service_args=service_args) as browser:
yield browser
def test_browser_activate_tab():
browser = pychrome.Browser()
tabs = []
for i in range(10):
tabs.append(browser.new_tab())
for tab in tabs:
browser.activate_tab(tab)
def close(host, port, secure, id):
"""close a target/tab by id"""
url = "%s://%s:%s" % ("https" if secure else "http", host, port)
try:
browser = pychrome.Browser(url)
click.echo(browser.close_tab(id))
except Exception as e:
click.echo(e)
def list(host, port, secure):
"""list all the available targets/tabs"""
url = "%s://%s:%s" % ("https" if secure else "http", host, port)
try:
browser = pychrome.Browser(url)
click.echo(json.dumps(browser.list_tab(), cls=JSONTabEncoder,
indent=4))
except Exception as e:
click.echo(e)
def __init__(self):
# google-chrome --headless --disable-gpu --remote-debugging-port=9222
# os.system("F:\\software\\Google\\Chrome\\Application\\chrome.exe --headless --disable-gpu --remote-debugging-port=9222")
# cc = os.popen("F:\\software\\Google\\Chrome\\Application\\chrome.exe --headless --disable-gpu --remote-debugging-port=9222")
# create a browser instance
self.browser = pychrome.Browser(url="http://127.0.0.1:9222")
self.url = "http://quote.eastmoney.com/concept/"
self.result = []
def test_set_wrong_listener():
browser = pychrome.Browser()
tab = browser.new_tab()
try:
tab.Network.requestWillBeSent = "test"
assert False, "never get here"
except pychrome.RuntimeException:
pass
def test_browser_tabs_map():
browser = pychrome.Browser()
tab = browser.new_tab()
assert tab in browser.list_tab()
assert tab in browser.list_tab()
browser.close_tab(tab)
assert tab not in browser.list_tab()
def version(host, port, secure):
"""show the browser version"""
url = "%s://%s:%s" % ("https" if secure else "http", host, port)
try:
browser = pychrome.Browser(url)
click.echo(json.dumps(browser.version(), indent=4))
except Exception as e:
click.echo(e)
def test_invalid_method():
browser = pychrome.Browser()
tab = browser.new_tab()
try:
tab.Page.NotExistMethod()
assert False, "never get here"
except pychrome.CallMethodException:
pass
def new(host, port, secure, url="about:blank"):
"""create a new target/tab"""
_url = "%s://%s:%s" % ("https" if secure else "http", host, port)
try:
browser = pychrome.Browser(_url)
click.echo(
json.dumps(browser.new_tab(url), cls=JSONTabEncoder, indent=4))
except Exception as e:
click.echo(e)
def __init__(self, *args, **kwargs):
# create a browser instance
self.browser = pychrome.Browser(url="http://127.0.0.1:9221")
# create a tab
self.tab = self.browser.new_tab()
#self.tab.Network.requestWillBeSent = self.request_will_be_sent
# start the tab
self.tab.start()
# call method
self.tab.Network.enable()
def __init__(self, log_id, port_number, agent_name, browser_pid):
self.agent_name = agent_name
self.logger = logging.getLogger(log_id)
self.port = port_number
print "Devtools listening on this port: ", self.port
self.browser = pychrome.Browser(url="http://127.0.0.1:%s" % (self.port,))
# Contains EventHandler objects for each tab; tab_id => EventHandler obj
self.eh = {}
self.download_path = os.path.join(MAIN_LOG_PATH, RAW_DOWNLOADS_DIR)
self.last_window_open_url = None
self.browser_pid = browser_pid
def __init__(self, url="http://127.0.0.1:9222",
_request_will_be_sent=None,
_web_socket_created=None):
if _web_socket_created is None:
_web_socket_created = self.web_socket_created
if _request_will_be_sent is None:
_request_will_be_sent = self.request_will_be_sent
self.browser = pychrome.Browser(url=url)
self.tab = self.browser.new_tab()
self.tab.set_listener("Network.requestWillBeSent", _request_will_be_sent)
self.tab.set_listener("Network.webSocketFrameReceived", _web_socket_created)
def test_normal_callmethod():
browser = pychrome.Browser()
tab = browser.new_tab()
result = tab.Page.navigate(url="http://www.fatezero.org")
assert result['frameId']
time.sleep(1)
result = tab.Runtime.evaluate(expression="document.domain")
assert result['result']['type'] == 'string'
assert result['result']['value'] == 'www.fatezero.org'
def __init__(self, index, multiThreaded=False):
self.number = index
self.browser = pychrome.Browser()
tabs = self.browser.list_tab()
# if not tabs:
self.tab = self.browser.new_tab()
# else:
# self.tab = tabs[0]
# self.tab.debug = True
self.tab.start()
self.tab.Page.enable()
self.tab.Page.navigate(
url="http://www.foddy.net/Athletics.html?webgl=true")
if (multiThreaded):
self.tab.call_method("Page.startScreencast",
format="png",
quality=25,
everyNthFrame=5,
maxWidth=640,
maxHeight=400)
self.tab.Page.screencastFrame = self.processStreamFrame
# self.gameIntroTemplate = cv2.resize(cv2.imread('intro.png', 0), (0,0), fx=0.68, fy=0.68)
if multiThreaded:
self.gameIntroTemplate = cv2.imread('intro.png', 0)
self.gameLostTemplate = cv2.imread('lost.png', 0)
else:
self.gameIntroTemplate = cv2.imread('intro-small.png', 0)
self.gameLostTemplate = cv2.imread('lost-small.png', 0)
self.grayImage = None
self.image = None
self.scoreImage = None
self.imageIsSimilar = False
self.scoreIsSimilar = False
self.previousKey = None
# cv2.namedWindow(str(self.number))
# while True:
while self.image is None:
if not multiThreaded:
self.takeScreenshot()
time.sleep(1)
def test_callback_exception():
browser = pychrome.Browser()
tab = browser.new_tab()
def request_will_be_sent(**kwargs):
raise Exception("will not stop")
tab.Network.requestWillBeSent = request_will_be_sent
tab.Network.enable()
tab.Page.navigate(url="chrome://newtab/")
if tab.wait(timeout=3):
assert False, "never get here"
