def test_xhr_post_2(self):
intermediate_url = "http://localhost:{}/ignore-headers".format(
self.mock_server_port)
target_url = "http://localhost:{}/post/form_urlencoded".format(
self.mock_server_port)
with ChromeController.ChromeContext(CHROME_BINARY_NAME,
additional_options=[
'--no-sandbox',
'--disable-setuid-sandbox'
]) as cr:
# print(ret)
# print("")
first_nav = cr.blocking_navigate_and_get_source(intermediate_url)
# print("First nav to '%s'" % intermediate_url)
# print(first_nav)
# print("Doing XHR to '%s'" % target_url)
data = urllib.parse.urlencode({'test': 1, "moar": "two"})
ret = cr.xhr_fetch(target_url,
post_data=data,
post_type='application/x-www-form-urlencoded')
pprint.pprint(ret)
self.assertEqual(ret['response'], '{"oh" : "hai"}')
self.assertEqual(ret['code'], 200)
self.assertEqual(ret['mimetype'], "application/json")
def _init_chromium_instance(self):
# Current runners are configured to use 10 threads.
self.__cr = ChromeController.TabPooledChromium(binary=self.__bin_name,
tab_pool_max_size=5,
*self._args,
**self._kwargs)
self.__initialized = True
def test_slow_redirect_1(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME,
additional_options=[
'--no-sandbox',
'--disable-setuid-sandbox'
]) as cr:
inurl_3 = "http://localhost:{}/redirect_slow/from-1".format(
self.mock_server_port)
outurl_3 = "http://localhost:{}/redirect_slow/to-1".format(
self.mock_server_port)
cr.blocking_navigate_and_get_source(inurl_3)
for x in range(999):
time.sleep(1)
current_title, _ = cr.get_page_url_title()
if x > WRAPPER_STEP_THROUGH_TIMEOUT:
raise RuntimeError("Timed out!")
if 'Still at title?' not in current_title:
break
resp = cr.handle_page_location_changed()
_, nurl_3 = cr.get_page_url_title()
nurl2_3 = cr.get_current_url()
self.assertEqual(outurl_3, nurl_3)
self.assertEqual(outurl_3, nurl2_3)
self.assertEqual(resp['content'], "Slow-Redirect-end-1")
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "text/html")
def test_plain_instantiation_1(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME,
additional_options=[
'--no-sandbox',
'--disable-setuid-sandbox'
]) as cr:
self.assertTrue(cr is not None)
def test_redirect_handling_7(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME) as cr:
# This is a infinitely recursive redirect.
with self.assertRaises(ChromeController.ChromeResponseNotReceived):
inurl_6 = "http://localhost:{}/redirect/bad-3".format(
self.mock_server_port)
cr.blocking_navigate(inurl_6)
def test_cloudflare_auto(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME,
additional_options=[
'--no-sandbox',
'--disable-setuid-sandbox'
]) as cr:
tgturl = "http://127.0.0.1:{}/cloudflare_under_attack_shit".format(
self.mock_server_port)
cr.blocking_navigate_and_get_source(tgturl)
for x in range(999):
time.sleep(1)
current_title, _ = cr.get_page_url_title()
if x > WRAPPER_STEP_THROUGH_TIMEOUT:
raise RuntimeError("Timed out!")
if 'Just a moment...' not in current_title:
break
resp = cr.handle_page_location_changed()
content = cr.get_rendered_page_source()
self.assertEqual(
content,
'<html><head><title>At target CF page!</title></head><body>CF Redirected OK?</body></html>'
)
def test_title():
crbin = "google-chrome"
with ChromeController.ChromeContext(crbin) as cr:
cr.blocking_navigate("http://www.google.com", timeout=10)
print("Current URL:", cr.get_current_url())
print(cr.get_page_url_title())
def getItemChromium(self, itemUrl):
self.log.info("Fetching page for URL: '%s' with Chromium" % itemUrl)
with ChromeController.ChromeContext(self._cr_binary) as cr:
self._syncIntoChromium(cr)
response = cr.blocking_navigate_and_get_source(itemUrl, timeout=10)
raw_url = cr.get_current_url()
fileN = urllib.parse.unquote(
urllib.parse.urlparse(raw_url)[2].split("/")[-1])
fileN = bs4.UnicodeDammit(fileN).unicode_markup
self._syncOutOfChromium(cr)
# Probably a bad assumption
if response['binary']:
mType = "application/x-binary"
else:
mType = "text/html"
# So, self._cr.page_source appears to be the *compressed* page source as-rendered. Because reasons.
content = response['content']
return content, fileN, mType
def test_fetch_decode_1(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME) as cr:
# text/html content should be decoded automatically.
resp = cr.blocking_navigate_and_get_source("http://localhost:{}/html-decode".format(self.mock_server_port), timeout=TIMEOUT_SECS)
self.assertEqual(resp['content'], 'Root OK?')
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "text/html")
def fetch_check_headers(self, expect_headers):
try:
# Configure mock server.
self.mock_server_port, self.mock_server, self.mock_server_thread = testing_server.start_server(
self, expect_headers)
intermediate_url = "http://localhost:{}/ignore-headers".format(
self.mock_server_port)
target_url = "http://localhost:{}/json/valid".format(
self.mock_server_port)
with ChromeController.ChromeContext(CHROME_BINARY_NAME) as cr:
ret = cr.update_headers(expect_headers)
# print("update_headers return:")
# print(ret)
# print("")
first_nav = cr.blocking_navigate_and_get_source(
intermediate_url)
# print("First nav to '%s'" % intermediate_url)
# print(first_nav)
# print("Doing XHR to '%s'" % target_url)
ret = cr.xhr_fetch(target_url)
# pprint.pprint(ret)
self.assertEqual(ret['response'], '{"oh" : "hai"}')
self.assertEqual(ret['code'], 200)
self.assertEqual(ret['mimetype'], "application/json")
finally:
self.mock_server.shutdown()
def test_fetch_decode_json_1(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME) as cr:
resp = cr.blocking_navigate_and_get_source(
"http://localhost:{}/json/valid".format(self.mock_server_port),
timeout=TIMEOUT_SECS)
self.assertEqual(resp['content'], '{"oh" : "hai"}')
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "application/json")
def test_redirect_handling_4(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME) as cr:
inurl_3 = "http://localhost:{}/redirect/from-4".format(self.mock_server_port)
resp = cr.blocking_navigate_and_get_source(inurl_3, timeout=TIMEOUT_SECS)
self.assertEqual(resp['content'], 'Redirect-To-4')
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "text/html")
def test_fetch_compressed_2(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME) as cr:
resp = cr.blocking_navigate_and_get_source(
"http://localhost:{}/compressed/deflate".format(
self.mock_server_port),
timeout=TIMEOUT_SECS)
self.assertEqual(resp['content'], 'Root OK?')
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "text/html")
def test_rendered_fetch():
crbin = "google-chrome"
cr = ChromeController.ChromeRemoteDebugInterface(binary=crbin)
resp = cr.blocking_navigate(
"https://www.catatopatch.com/appraise-chapter-15", timeout=10)
print("Current URL:", cr.get_current_url())
rcnt = cr.get_rendered_page_source()
print("content:", type(rcnt))
def test_cycle():
crbin = "google-chrome"
for x in range(30):
print("Starting loop %s" % x)
with ChromeController.ChromeContext(crbin) as cr:
print("Looping:", x)
print(cr)
cr.blocking_navigate('http://www.google.com')
print("Deleted")
print("Ending loop %s" % x)
def test_basic_fetch_1(self):
# Configure mock server.
tgturl = "http://localhost:{}".format(self.mock_server_port)
with ChromeController.ChromeContext(CHROME_BINARY_NAME) as cr:
cr.update_headers({})
resp = cr.blocking_navigate_and_get_source(tgturl)
self.assertEqual(resp['content'], 'Root OK?')
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "text/html")
def test_get_head_1(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME) as cr:
inurl_1 = "http://localhost:{}/".format(self.mock_server_port)
cr.blocking_navigate(inurl_1)
_, nurl_1 = cr.get_page_url_title()
nurl2_1 = cr.get_current_url()
self.assertEqual(nurl2_1, inurl_1)
self.assertEqual(nurl_1, inurl_1)
def test_redirect_handling_6(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME,
additional_options=[
'--no-sandbox',
'--disable-setuid-sandbox'
]) as cr:
# This is a infinitely recursive redirect.
with self.assertRaises(ChromeController.ChromeResponseNotReceived):
inurl_6 = "http://localhost:{}/redirect/bad-2".format(
self.mock_server_port)
cr.blocking_navigate(inurl_6)
def test_redirect_handling_8(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME) as cr:
inurl_7 = "http://localhost:{}/redirect/from-5".format(self.mock_server_port)
# Assumes localhost seems to resolve to the listening address (here it's 0.0.0.0). Is this ever not true? IPv6?
outurl_7 = "http://127.0.0.1:{}/".format(self.mock_server_port)
cr.blocking_navigate(inurl_7)
_, nurl_7 = cr.get_page_url_title()
nurl2_7 = cr.get_current_url()
self.assertEqual(outurl_7, nurl_7)
self.assertEqual(outurl_7, nurl2_7)
def test_redirect_handling_3(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME) as cr:
inurl_3 = "http://localhost:{}/redirect/from-1".format(self.mock_server_port)
outurl_3 = "http://localhost:{}/redirect/to-1".format(self.mock_server_port)
cr.blocking_navigate(inurl_3)
_, nurl_3 = cr.get_page_url_title()
nurl2_3 = cr.get_current_url()
self.assertEqual(outurl_3, nurl_3)
self.assertEqual(outurl_3, nurl2_3)
def test_fetch_1(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME,
additional_options=[
'--no-sandbox',
'--disable-setuid-sandbox'
]) as cr:
resp = cr.blocking_navigate_and_get_source(
"http://localhost:{}".format(self.mock_server_port),
timeout=TIMEOUT_SECS)
self.assertEqual(resp['content'], 'Root OK?')
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "text/html")
def test_fetch_decode_json_3(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME,
additional_options=[
'--no-sandbox',
'--disable-setuid-sandbox'
]) as cr:
resp = cr.blocking_navigate_and_get_source(
"http://localhost:{}/json/invalid".format(
self.mock_server_port),
timeout=TIMEOUT_SECS)
self.assertEqual(resp['content'], 'LOLWAT')
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "application/json")
def test_redirect_handling_3(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME,
additional_options=[
'--no-sandbox',
'--disable-setuid-sandbox'
]) as cr:
inurl_3 = "http://localhost:{}/redirect/from-3".format(
self.mock_server_port)
resp = cr.blocking_navigate_and_get_source(inurl_3,
timeout=TIMEOUT_SECS)
self.assertEqual(resp['content'], 'Redirect-To-3')
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "text/html")
def chromiumGetRenderedItem(self, url):
with ChromeController.ChromeContext(self._cr_binary) as cr:
self._syncIntoChromium(cr)
# get_rendered_page_source
cr.blocking_navigate(url)
content = cr.get_rendered_page_source()
mType = 'text/html'
fileN = ''
self._syncOutOfChromium(cr)
return content, fileN, mType
def test_get_head_1(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME,
additional_options=[
'--no-sandbox',
'--disable-setuid-sandbox'
]) as cr:
inurl_1 = "http://localhost:{}/".format(self.mock_server_port)
cr.blocking_navigate(inurl_1)
_, nurl_1 = cr.get_page_url_title()
nurl2_1 = cr.get_current_url()
self.assertEqual(nurl2_1, inurl_1)
self.assertEqual(nurl_1, inurl_1)
def fetch_check_headers(self, expect_headers):
try:
# Configure mock server.
self.mock_server_port, self.mock_server, self.mock_server_thread = testing_server.start_server(
self, expect_headers)
tgturl = "http://localhost:{}".format(self.mock_server_port)
with ChromeController.ChromeContext(CHROME_BINARY_NAME) as cr:
cr.update_headers(expect_headers)
resp = cr.blocking_navigate_and_get_source(tgturl)
self.assertEqual(resp['content'], 'Root OK?')
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "text/html")
finally:
self.mock_server.shutdown()
def getHeadChromium(self, url, referrer=None):
self.log.info("Getting HEAD with Chromium")
if not referrer:
referrer = url
with ChromeController.ChromeContext(self._cr_binary) as cr:
self._syncIntoChromium(cr)
cr.blocking_navigate(referrer)
time.sleep(random.uniform(2, 6))
cr.blocking_navigate(url)
dummy_title, cur_url = cr.get_page_url_title()
self._syncOutOfChromium(cr)
return cur_url
def test_tabs():
crbin = "google-chrome"
for x in range(30):
with ChromeController.ChromeContext(binary="google-chrome") as cr:
print("Context manager entered")
tabl = [cr.new_tab(), cr.new_tab(), cr]
print("Tabs:", tabl)
print("Transport:")
print(tabl[0].transport)
# cr.blocking_navigate("http://www.google.com", timeout=10)
print("Loop")
for idx, tab in enumerate(tabl):
print("Fetching using tab %s -> %s" % (idx, tab))
tab.blocking_navigate("http://www.google.com", timeout=10)
print("Complete")
def test():
ua = dict(WebRequest.getUserAgent())
# print(ua)
crbin = os.path.abspath("../Chromium/src/out/Headless/headless_shell")
cr = ChromeController.ChromeRemoteDebugInterface(crbin)
# print(cr)
resp = cr.Emulation_setVisibleSize(1500, 1000)
# print("Viewport size", resp)
# resp = cr.update_headers(ua)
# print("Set extra headers: ", resp)
resp = cr.blocking_navigate("http://www.whatarecookies.com/cookietest.asp", timeout=10)
cooks1 = cr.get_cookies()
resp = cr.blocking_navigate("http://google.com", timeout=10)
cooks2 = cr.get_cookies()
print()
print("Cookies after first 'Page.navigate' command:")
for cookie in cooks1:
print(' ', cookie)
print()
print("Cookies after second 'Page.navigate' command:")
for cookie in cooks2:
print(' ', cookie)
# for cook in cooks1:
# ret = cr.set_cookie(cook)
# print(ret)
# print()
# print("Reinstated cookies:")
# cooks3 = cr.get_cookies()
# for cookie in cooks3:
# print(cookie)
wait_time = 5
for x in range(wait_time):
data = cr.drain_transport()
# pprint.pprint(data)
print("Sleeping: ", wait_time-x)
def test_multi_redirect_handling_1(self):
with ChromeController.ChromeContext(binary=CHROME_BINARY_NAME) as cr:
inurl_3 = "http://localhost:{}/redirect_mult/from-1".format(
self.mock_server_port)
outurl_3 = "http://localhost:{}/redirect_mult/to-5".format(
self.mock_server_port)
resp = cr.blocking_navigate_and_get_source(inurl_3)
_, nurl_3 = cr.get_page_url_title()
nurl2_3 = cr.get_current_url()
self.assertEqual(outurl_3, nurl_3)
self.assertEqual(outurl_3, nurl2_3)
self.assertEqual(resp['content'], 'Multi-Redirect-end-5')
self.assertEqual(resp['binary'], False)
self.assertEqual(resp['mimetype'], "text/html")