def _setup_proxy_server(self,
downstream_kbps=None,
upstream_kbps=None,
latency=None):
server = browsermobproxy.Server(BROWSERMOB_PROXY_PATH)
server.start()
proxy = server.create_proxy()
# The proxy server is pretty sluggish, setting the limits might not
# achieve the desired behavior.
proxy_options = {}
if downstream_kbps:
proxy_options['downstream_kbps'] = downstream_kbps
if upstream_kbps:
proxy_options['upstream_kbps'] = upstream_kbps
if latency:
proxy_options['latency'] = latency
if len(proxy_options.items()) > 0:
proxy.limits(proxy_options)
return server, proxy
def crawl_cryptocompare(self):
path_to_bin = os.path.join('C:\\', 'Users', 'Gobi', 'Downloads',
'browsermob-proxy-2.1.4', 'bin',
'browsermob-proxy')
server = browsermobproxy.Server(path_to_bin)
server.start()
proxy = server.create_proxy()
options = selenium.webdriver.ChromeOptions()
options.add_argument('--proxy-server=%s' % proxy.proxy)
driver = selenium.webdriver.Chrome(chrome_options=options)
proxy.new_har(
options={
'captureHeaders': True,
'captureContent': True,
'captureBinaryContent': True
})
time.sleep(5)
try:
driver.get(self.url)
proxy.wait_for_traffic_to_stop(1, 10)
time.sleep(10)
entries = (Entry(entry) for entry in proxy.har['log']['entries']
if 'streamer' in entry['request']['url'])
self.entries = [entry for entry in entries if entry.trades]
except Exception as e:
print(e)
driver.close()
server.stop()
time.sleep(5)
def _setup_proxy_server(self, downstream_kbps=None, upstream_kbps=None,
latency=None):
"""Sets up a browsermobproxy server.
Args:
downstream_kbps: int. The downstream speed in kbps. Defaults to
None.
upstream_kbps: int. The upstream speed in kbps. Defaults to None.
latency: int. The latency of the server in ms. Defaults to None.
Returns:
tuple(Server, dict): A tuple consisting of the Server object and
the proxy as a pair.
"""
server = browsermobproxy.Server(BROWSERMOB_PROXY_PATH)
server.start()
proxy = server.create_proxy()
# The proxy server is pretty sluggish, setting the limits might not
# achieve the desired behavior.
proxy_options = {}
if downstream_kbps:
proxy_options['downstream_kbps'] = downstream_kbps
if upstream_kbps:
proxy_options['upstream_kbps'] = upstream_kbps
if latency:
proxy_options['latency'] = latency
if len(list(proxy_options.items())) > 0:
proxy.limits(proxy_options)
return server, proxy
def proxy_server():
server = browsermobproxy.Server()
server.start()
try:
yield server.create_proxy()
finally:
server.stop()
def pre_execution(browser, foreground):
"""Handles initialization of global resources"""
global SERVER
global DISPLAY
if "bmp" in common.CONFS[browser]:
SERVER = browsermobproxy.Server()
SERVER.start()
logging.debug("Browsermob started")
if "xvfb" in common.CONFS[browser] and not foreground:
DISPLAY = Xvfb(width=1280, height=720)
DISPLAY.start()
logging.debug("Xvfb started")
setup_signals()
def startProxyServer(
Location_Of_Browser_Mob_Proxy=Location_Of_Browser_Mob_Proxy,
Proxy_Server_Options={}):
'''
Starts The Browser Mob Proxy Server
Location_Of_Browser_Mob_Proxy : Absolute Path Of The Browser Mob Proxy Batch File
Server_Options : Options For The Browser Mob Proxy Server Object
Returns : Browser Mob Proxy Server Object If The Server Is Created Successfully Else None
'''
try:
Server = browsermobproxy.Server(Location_Of_Browser_Mob_Proxy,
Proxy_Server_Options)
Server.start()
except Exception, e:
print 'Start Proxy Server Method'
print 'Exception Occured:', e.__class__
print 'Exception Message:', str(e)