def init_browser(self):
# response listen proxy
server = Server(r'C:\python_job\khedu-test\browsermob-proxy-2.1.4\bin\browsermob-proxy.bat')
server.start()
proxy = server.create_proxy()
proxy.new_har(options={
'captureContent': True,
'captureHeaders': True
})
# -- chromedriver --
chrome_options = Options()
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--proxy-server={0}'.format(proxy.proxy))
chrome_options.add_argument('--headless') # 無頭模式
browser = webdriver.Chrome(r'C:\python_job\khedu-test\chromedriver.exe', options=chrome_options)
browser.maximize_window()
time.sleep(1)#變大視窗需要等待
browser.set_page_load_timeout(30)
wait = WebDriverWait(browser,30)
browser.get(login_url)
# -- login --
id0 = pq(browser.page_source)('div.z-page').attr('id').replace('_', '') # 每次重load網站都有個隨機id
browser.find_element_by_xpath('//*[@id="{}b"]'.format(id0)).send_keys(username)
browser.find_element_by_xpath('//*[@id="{}c"]'.format(id0)).send_keys(password)
browser.find_element_by_xpath('//*[@id="{}g"]'.format(id0)).click()
time.sleep(2) # not so fast
browser.refresh() #頁面會卡在轉圈圈 需要reload
time.sleep(2)
return browser, proxy, wait
def initializeChrome():
global driver
global server
global proxy
dict = {'port': 8090}
path_to_bmp = Path(
"./binaries/browsermob-proxy-2.1.4/bin/browsermob-proxy").absolute()
path_on_windows = str(PureWindowsPath(path_to_bmp))
server = Server(path=path_on_windows, options=dict)
#server = Server(path="binaries/browsermob-proxy-2.1.4/bin/browsermob-proxy", options=dict)
server.start()
proxy = server.create_proxy()
user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.50 Safari/537.36'
username_windows = input(
"Please enter your username the way it's written in C:/Users/ - it's needed to locate your Chrome user data."
)
user_data_path = Path('C:/Users/' + str(username_windows) +
'/AppData/Local/Google/Chrome/User Data')
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--proxy-server={0}".format(proxy.proxy))
chrome_options.add_experimental_option("excludeSwitches",
['enable-automation'])
chrome_options.add_argument(f'--user-data-dir={user_data_path}')
path_to_chromedriver = str(Path("./binaries/chromedriver.exe").absolute())
driver = webdriver.Chrome(path_to_chromedriver, options=chrome_options)
print('initialized Chrome window!')
def setUp(self):
"""
Start the browser with a browsermob-proxy instance for use by the test.
You *must* call this in the `setUp` method of any subclasses before using the browser!
Returns:
None
"""
try:
# Start server proxy
server = Server('browsermob-proxy')
server.start()
self.proxy = server.create_proxy()
proxy_host = os.environ.get('BROWSERMOB_PROXY_HOST', '127.0.0.1')
self.proxy.remap_hosts('localhost', proxy_host)
except:
self.skipTest('Skipping: could not start server with browsermob-proxy.')
# parent's setUp
super(WebAppPerfReport, self).setUp()
# Initialize vars
self._page_timings = []
self._active_har = False
self._with_cache = False
# Add one more cleanup for the server
self.addCleanup(server.stop)
def __start_bmproxy(self, config):
# BrowserMob
from browsermobproxy import Server
from arjuna.tpi.constant import ArjunaOption
capture_traffic = config.value(ArjunaOption.BROWSER_NETWORK_RECORDER_ENABLED)
if capture_traffic:
bmproxy_dir = config.value(ArjunaOption.TOOLS_BMPROXY_DIR)
sub_dirs = os.listdir(bmproxy_dir)
bmproxy_bin_dir = None
if "bin" in sub_dirs:
bmproxy_bin_dir = os.path.join(bmproxy_dir, "bin")
else:
sub_dirs.sort(reverse=True) # Last version will be picked.
for d in sub_dir:
if d.startswith("browsermob"):
bmproxy_bin_dir = os.path.join(bmproxy_dir, d, "bin")
break
if bmproxy_bin_dir is None:
raise Exception("Network recording is enabled in configuration. There was an error in creating proxy server/server using BrowserMob Proxy. Could not find proxy package at {}".format(bmproxy_dir))
if platform.system().lower() == "windows":
exe = "browsermob-proxy.bat"
else:
exe = "browsermob-proxy"
bmproxy_exe_path = os.path.join(bmproxy_bin_dir, exe)
try:
self.__bmproxy_server = Server(bmproxy_exe_path)
self.__bmproxy_server.start()
except ProxyServerError as e:
raise Exception("Network recording is enabled in configuration. There was an error in creating proxy server/server using BrowserMob Proxy. Fix and retry. Error message: {}".format(str(e)))
class testAuthBasic(unittest.TestCase):
def setUp(self):
self.server = Server(
r"C:\Users\user\Documents\ScrinShots\SeleniumLessons\basic_auth\browsermob-proxy\bin\browsermob-proxy"
)
self.server.start()
self.proxy = self.server.create_proxy()
self.profile = webdriver.FirefoxProfile()
self.profile.set_proxy(self.proxy.selenium_proxy())
self.driver = webdriver.Firefox(firefox_profile=self.profile)
self.driver.implicitly_wait(8)
self.driver.get("https://auth-demo.aerobatic.io/")
def tearDown(self):
self.server.stop()
self.driver.quit()
def test_search(self):
self.proxy.basic_authentication("auth-demo.aerobatic.io", "aerobatic",
"aerobatic")
self.driver.get("https://auth-demo.aerobatic.io/protected-standard/")
assert self.driver.find_element_by_css_selector(
"a.button-primary[href='/']")
class GetWebPageHar:
proxy_path = "./browsermob-proxy-2.1.4/bin/browsermob-proxy"
def __init__(self):
self.server = Server(str(Path(self.proxy_path).absolute()))
self.server.start()
self.proxy = self.server.create_proxy()
def get_har(self, url, output_filename):
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--proxy-server={0}".format(
self.proxy.proxy))
chrome_options.add_argument("--ignore-certificate-errors")
chrome_options.add_argument("--incognito")
driver = webdriver.Chrome(chrome_options=chrome_options)
self.proxy.new_har("google")
driver.get(url)
time.sleep(10)
har_json = json.dumps(self.proxy.har, indent=4, ensure_ascii=False)
with open(output_filename, "w") as f:
f.write(har_json)
driver.quit()
def stop(self):
self.server.stop()
def __init__(self, results, reports, **kwargs):
self.results = results
self.reports = reports
self.args = kwargs
self.listen_port = 9760
self.lock = threading.Lock()
self.vulnerable = []
self.server = None
self.cookies = {}
for entry in self.args['cookie'].split(';'):
if entry.find('=') == -1:
continue
key, value = entry.strip().split('=', 1)
self.cookies[key] = value
# Create proxy server
logging.info('Starting browsermobproxy server...')
self.proxy_server = Server(self.args['browsermobproxy'])
self.proxy_server.start()
self.proxy = self.proxy_server.create_proxy()
logging.info('Browsermobproxy server started')
# Create Chrome engine
logging.info('Creating Selenium Chrome webdriver...')
self.chrome_options = webdriver.ChromeOptions()
self.chrome_options.add_argument('--proxy-server={}'.format(
self.proxy.proxy))
if 'headless' in self.args:
self.chrome_options.add_argument('--headless')
self.chrome_options.add_argument('--disable-gpu')
self.chrome_options.add_argument("--disable-extensions")
self.driver = webdriver.Chrome(chrome_options=self.chrome_options)
logging.info('Selenium Chrome webdriver created')
def reloadHeaderAndCookie():
global httpArchive
browsermob_path = '/usr/local/browsermob-proxy-2.1.4/bin/browsermob-proxy'
server = Server(browsermob_path)
proxy = server.create_proxy()
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--proxy-server={0}".format(proxy.proxy))
browser = webdriver.Chrome(options=chrome_options)
options = {'captureHeaders': True, 'captureCookies': True}
signIn(browser)
proxy.new_har("lifemiles", options=options)
url = "https://www.lifemiles.com/fly/find"
time.sleep(2)
browser.get(url)
time.sleep(5)
originID = "ar_bkr_fromairport"
browser.find_element_by_id(originID).send_keys("SFO")
time.sleep(2)
destinationID = "ar_bkr_toairport"
browser.find_element_by_id(destinationID).send_keys("SFO")
time.sleep(2)
server.start()
searchClass = "Booker_bookerActionButtonSmall__3Fh2d"
try:
browser.find_element_by_class_name(searchClass).click()
except Exception, e:
print("was not able to click search button, try javascript")
browser.execute_script(
"document.querySelector('.{0}').click()".format(searchClass))
class ProxyDistributor:
browsermob_proxy_path = "/home/amerigo/PycharmProjects/Progetto/tesi/risorse/browsermob-proxy/bin/browsermob-proxy"
def __init__(self):
self.server = Server(self.browsermob_proxy_path)
self.actual_port = 9090
def get_new_proxy(self):
proxy = self.server.create_proxy({
'port': self.actual_port,
'captureHeaders': True,
'captureContent': True,
'captureBinaryContent': True
})
self.actual_port += 1
return proxy
def start(self):
if self.server is None:
self.server = Server(self.browsermob_proxy_path)
self.server.start()
def stop(self):
if self.server is not None:
self.server.stop()
def __init__(self, url, username, password):
self.bearer = None
# enable browser logging
d = DesiredCapabilities.CHROME
d['loggingPrefs'] = {'browser': 'ALL'}
self.server = Server("BrowserMob\\bin\\browsermob-proxy")
cli.print_info("Starting proxy server...")
self.server.start()
self.proxy = self.server.create_proxy()
cli.add_to_print("OK\n\n")
cli.print_warning("NOTE: Connections will show as INSECURE!\n")
options = webdriver.ChromeOptions()
self.url = url
options.add_argument("--proxy-server={0}".format(self.proxy.proxy))
cli.print_info("Browser started\n")
self.browser = Chrome(options=options,
executable_path="./chromedriver.exe",
desired_capabilities=d)
self.username = username
self.password = password
del password
del username
def setupdevices():
"""
Description:
Sets u browser proxy, Selenium driver, and har object
Usage:
[driver,proxy]=setupdevices()
Inputs:
NA
Output:
Selenium driver
Browsermob proxy
Browsermob server
"""
#set up proxy
server = Server("############/browsermob-proxy-2.0-beta-9/bin/browsermob-proxy")
server.start()
proxy = server.create_proxy()
profile = webdriver.FirefoxProfile()
profile.set_proxy(proxy.selenium_proxy())
proxy.new_har("________")
#set up driver
driver = webdriver.Firefox(firefox_profile=profile)
return (driver,proxy,server)
def create_server(browser, websites):
print("=====> Creating New Server - Please Wait... <=====")
server = Server(browsermob_proxy_path)
server.start()
proxy = server.create_proxy()
configure_server(proxy, browser, websites)
close_server(proxy, server)
class ProxyManager:
__BMP = filename
def __init__(self):
self.__server = Server(ProxyManager.__BMP)
self.__client = None
def start_server(self):
self.__server.start()
print('start proxy server')
return self.__server
def start_client(self):
self.__client = self.__server.create_proxy(
params={"trustAllServers": "true"})
print('start proxy client')
return self.__client
@property
def client(self):
return self.__client
@property
def server(self):
return self.__server
def test2():
browsermob_path = '/usr/local/browsermob-proxy-2.1.4/bin/browsermob-proxy'
server = Server(browsermob_path, {'port':9999})
time.sleep(1)
proxy = server.create_proxy()
time.sleep(1)
def get_driver(self):
# server = Server(r'/home/reggie/Documents/app/browsermob-proxy-2.1.4/bin/browsermob-proxy')
server = Server(r'/root/browsermob-proxy-2.1.4/bin/browsermob-proxy')
server.start()
print('server started')
# map={'http':'http://officepx.datayes.com','https':'http://officepx.datayes.com'}
proxy = server.create_proxy()
print('proxy created')
caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "none"
chrome_options = Options()
chrome_options.add_argument('--ignore-ssl-errors=yes')
chrome_options.add_argument('--ignore-certificate-errors')
prefs = {'profile.managed_default_content_settings.images': 2}
chrome_options.add_experimental_option('prefs', prefs)
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_experimental_option('excludeSwitches',
['enable-automation']) # 以开发者模式
chrome_options.add_argument('--proxy-server={0}'.format(proxy.proxy))
drive = webdriver.Chrome(options=chrome_options,
desired_capabilities=caps)
print('driver opened')
return drive, server, proxy
def get_driver(request, cmdopt_browser, cmdopt_window):
"""Fixture to create, return and close driver"""
server = Server(
"C:\\Program Files (x86)\\browsermob-proxy\\bin\\browsermob-proxy",
{"port": 9090})
server.start()
proxy = server.create_proxy()
url = urllib.parse.urlparse(proxy.proxy).path
driver = None
if cmdopt_browser == "ie":
driver = webdriver.Ie()
elif cmdopt_browser == "firefox":
if cmdopt_window == "headless":
options = Firefox_options()
options.add_argument("--headless")
options.add_argument('--proxy-server={}'.format(url))
driver = webdriver.Firefox(firefox_options=options)
else:
options = Firefox_options()
options.add_argument('--proxy-server={}'.format(url))
driver = webdriver.Firefox()
proxy.new_har()
request.addfinalizer(driver.quit)
elif cmdopt_browser == "chrome":
if cmdopt_window == "headless":
options = Chrome_options()
options.headless = True
options.add_argument('--proxy-server={}'.format(url))
driver = webdriver.Chrome(chrome_options=options)
proxy.new_har()
request.addfinalizer(driver.quit)
else:
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server={}'.format(url))
d = DesiredCapabilities.CHROME
d['loggingPrefs'] = {'browser': 'ALL'}
driver = webdriver.Chrome(desired_capabilities=d,
chrome_options=chrome_options)
proxy.new_har()
ef_driver = EventFiringWebDriver(driver, MyListener())
def fin():
log_timestamp = str(datetime.datetime.now())[0:-4].replace(
'-', '.').replace(' ', '_').replace(':', '.')
browserlog_filename = log_timestamp + '_browser_log_file.log'
browserlogfile = open(browserlog_filename, 'w')
print('-------------------------')
for i in ef_driver.get_log('browser'):
print(i)
browserlogfile.write(str(i) + '\n')
print(proxy.har)
server.stop()
ef_driver.quit
request.addfinalizer(fin)
return ef_driver
else:
return "unsupported browser"
return driver
class ProxyManager:
__BMP = './browsermob-proxy-2.1.4/bin/browsermob-proxy'
def __init__(self):
self.__server = Server(ProxyManager.__BMP)
self.__client = None
def start_server(self):
self.__client = self.__server.create_proxy(
params={'trustAllServers': 'true'})
self.__server.start()
return self.__server
def start_client(self):
return self.__client
@property
def client(self):
return self.__client
@property
def server(self):
return self.__server
class BrowserMobProxyTestCaseMixin(object):
def __init__(self, *args, **kwargs):
self.browsermob_server = None
self.browsermob_port = kwargs.pop('browsermob_port')
self.browsermob_script = kwargs.pop('browsermob_script')
def setUp(self):
options = {}
if self.browsermob_port:
options['port'] = self.browsermob_port
if not self.browsermob_script:
raise ValueError('Must specify --browsermob-script in order to '
'run browsermobproxy tests')
self.browsermob_server = Server(self.browsermob_script,
options=options)
self.browsermob_server.start()
def create_browsermob_proxy(self):
client = self.browsermob_server.create_proxy()
with self.marionette.using_context('chrome'):
self.marionette.execute_script("""
Services.prefs.setIntPref('network.proxy.type', 1);
Services.prefs.setCharPref('network.proxy.http', 'localhost');
Services.prefs.setIntPref('network.proxy.http_port', %(port)s);
Services.prefs.setCharPref('network.proxy.ssl', 'localhost');
Services.prefs.setIntPref('network.proxy.ssl_port', %(port)s);
""" % {"port": client.port})
return client
def tearDown(self):
if self.browsermob_server:
self.browsermob_server.stop()
self.browsermob_server = None
__del__ = tearDown
class BaseFramework(object):
def __init__(self):
self.server = Server('./browsermob-proxy-2.1.4/bin/browsermob-proxy')
self.server.start()
self.proxy = self.server.create_proxy()
chrome_options = Options()
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--proxy-server={0}'.format(self.proxy.proxy))
self.browser = webdriver.Chrome(options=chrome_options)
def process_request(self, request, response):
pass
def process_response(self, response, request):
pass
def run(self, func, *args):
self.proxy.new_har(options={
'captureContent': True,
'captureHeaders': True
})
func(*args)
result = self.proxy.har
for entry in result['log']['entries']:
request = entry['request']
response = entry['response']
self.process_request(request, response)
self.process_response(response, request)
def __del__(self):
self.proxy.close()
self.browser.close()
def __init__(self,
url,
har_name,
browsermob_proxy_location,
selector_dictionary=None,
default_timeout=None,
firefox_binary=None,
highlight=False,
geckodriver="geckodriver"):
self.selector_dictionary = selector_dictionary
self.default_timeout = default_timeout if default_timeout is not None else 30
self.highlight = highlight
for proc in psutil.process_iter():
# check whether the process name matches
if proc.name() == "browsermob-proxy":
proc.kill()
options = {'port': 8090}
# self.server = Server(path="../tools/browsermob-proxy-2.1.4/bin/browsermob-proxy", options=dict)
self.server = Server(path=browsermob_proxy_location, options=options)
self.server.start()
time.sleep(1)
self.proxy = self.server.create_proxy()
time.sleep(1)
profile = webdriver.FirefoxProfile()
selenium_proxy = self.proxy.selenium_proxy()
profile.set_proxy(selenium_proxy)
self.driver = webdriver.Firefox(firefox_profile=profile,
firefox_binary=firefox_binary,
executable_path=geckodriver)
self.proxy.new_har(har_name)
self.driver.get(url)
def login(weburl, user, passwd, proxy_location, webdriver_location):
server = Server(proxy_location + r'\browsermob-proxy.bat')
server.start()
proxy = server.create_proxy()
proxy.new_har(options={'captureContent': True, 'captureHeaders': True})
chrome_options = Options()
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--proxy-server={}'.format(proxy.proxy))
browser = webdriver.Chrome(webdriver_location + r'\chromedriver',
options=chrome_options)
browser.set_page_load_timeout(30)
wait = WebDriverWait(browser, 10)
#登入
browser.get(login_url)
id0 = pq(browser.page_source)('div.z-page').attr('id').replace(
'_', '') #抓第一行 每個id，都會隨機亂產生
browser.find_element_by_xpath('//*[@id="{}b"]'.format(id0)).send_keys(
username) #亂數+b
browser.find_element_by_xpath('//*[@id="{}c"]'.format(id0)).send_keys(
password) #亂數+c
browser.find_element_by_xpath('//*[@id="{}g"]'.format(id0)).click() #亂數+g
time.sleep(4)
content_id = pq(browser.page_source)('div.z-page').attr('id').replace(
'_', '') #主頁面的亂數
print(id0)
print(content_id)
#回傳 browser, wait ,webid, proxy
return browser, wait, proxy, content_id
def reloadHeaderAndCookie():
browsermob_path = '/usr/local/browsermob-proxy-2.1.4/bin/browsermob-proxy'
server = Server(browsermob_path, {'port': 9999})
time.sleep(1)
proxy = server.create_proxy()
time.sleep(1)
def __init__(self, root="C:/xampp/htdocs/webscrape/", folder=""):
self.url = ""
self.root = root + folder + "/"
# Start browsermob proxy
self.server = Server(
r"C:\webdrivers\browsermob-proxy\bin\browsermob-proxy")
self.server.start()
self.proxy = self.server.create_proxy()
# Setup Chrome webdriver - note: does not seem to work with headless On
options = webdriver.ChromeOptions()
options.binary_location = r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe"
options.add_argument(
'--proxy-server=%s' % self.proxy.proxy
) # Setup proxy to point to our browsermob so that it can track requests
self.w = webdriver.Chrome(r'C:/webdrivers/chromedriver.exe',
chrome_options=options)
self.proxy.new_har("Listener", options={'captureHeaders':
True}) # Request listener
#self.proxy.new_har("Listener" ) # Request listener
print("Browser and Server initialized...")
def _start_ProxyHelper(self, options, proxy_port=None):
"""
启动代理类,用于监听Http请求
:param options: chrom浏览器启动选项
:param proxy_port: 代理端口号
"""
assert self._proxyHelper is None, "代理已开启"
# 代理端口,若未指定,获取默认端口
proxy_port = GF.proxy_port() if proxy_port is None else proxy_port
# 启动代理浏览器
server = Server(GF.proxy_path(), {'port': proxy_port})
server.start({'log_path': GF.cache_path()})
proxy = server.create_proxy()
# 开启har监听
proxy.new_har(options={'captureContent': True, 'captureHeaders': True})
options.add_argument('--proxy-server={0}'.format(proxy.proxy))
# 启动代理日志
self.init_proxy_log()
# 启动代理助手
self._proxyHelper = ProxyHelper(self, proxy)
return options
def __init__(self):
self.server = Server('./browsermob-proxy-2.1.4/bin/browsermob-proxy')
self.server.start()
self.proxy = self.server.create_proxy()
chrome_options = Options()
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--proxy-server={0}'.format(self.proxy.proxy))
self.browser = webdriver.Chrome(options=chrome_options)
def get_driver(self, browser, start_beacon_url):
server = Server(BROWSERMOB_LOCATION)
server.start()
self.proxy = server.create_proxy()
driver = webdriver.Firefox(proxy=self.proxy.selenium_proxy())
self.proxy.new_har()
self.beacon_url = start_beacon_url
return driver
def __init__(self, headless: bool = True):
super().__init__(headless=headless)
ProxyFirefoxRunner._force_shutdown_of_browsermob_proxy()
sleep(1)
self.proxy_server = Server(BROWSERMOB_PROXY)
self.proxy_server.start()
sleep(1)
self.proxy_client = self.proxy_server.create_proxy()
sleep(1)
def test():
browsermob_path = '/usr/local/browsermob-proxy-2.1.4/bin/browsermob-proxy'
server = Server(browsermob_path, {'port': 8090})
server.start()
time.sleep(1)
proxy = server.create_proxy()
time.sleep(1)
print("we started proxy")
def get_proxy_and_server(passed_dict):
print('Getting Browsermob proxy and server')
server = Server('qa/env/bin/browsermob-proxy/bin/browsermob-proxy',
options={'existing_proxy_port_to_use': 8999})
server.start()
proxy = server.create_proxy()
proxy.headers({"Authorization": passed_dict["Authorization"]})
proxy.new_har(HOST)
return proxy, server
def proxy():
"""Фикстура для конфигурирования прокси-сервера"""
server = Server(
"03_PageObject/browsermob-proxy-2.1.4/bin/browsermob-proxy")
server.start()
client = Client("localhost:8080")
# print(client.port)
client.new_har()
return server, client
def __init__(self, path=None):
if path is None:
self.server = Server(config['HAR']['SERVER_PATH'])
self.server.start()
self.proxy = self._start_proxy()
self.driver = self._start_chrome_driver()
self.df = None
else:
self.df = pd.read_csv(f'{path}')
def initProxy(self):
"""
step 4 初始化 browermobproxy
设置需要屏蔽的网络连接，此处屏蔽了css，和图片（有时chrome的设置会失效），可加快网页加载速度
新建proxy代理地址
"""
self.server = Server(self.PROXY_PATH)
self.server.start()
self.proxy = self.server.create_proxy()
def pytest_runtest_setup(item):
logger = logging.getLogger(__name__)
if item.config.option.bmp_test_proxy and 'skip_browsermob_proxy' not in item.keywords:
if hasattr(item.session.config, 'browsermob_server'):
server = item.session.config.browsermob_server
else:
server = Server(item.config.option.bmp_path, {'port': int(item.config.option.bmp_port)})
item.config.browsermob_test_proxy = server.create_proxy()
logger.info('BrowserMob test proxy started (%s:%s)' % (item.config.option.bmp_host, item.config.browsermob_test_proxy.port))
configure_browsermob_proxy(item.config.browsermob_test_proxy, item.config)
#TODO make recording of har configurable
item.config.browsermob_test_proxy.new_har()
def init_proxy_server(self, port=None):
kwargs = {}
if port is not None:
kwargs['port'] = port
if self.chained_proxy is not None:
if self.is_https:
kwargs['httpsProxy'] = self.chained_proxy
else:
kwargs['httpProxy'] = self.chained_proxy
if self.proxy_username is not None:
kwargs['proxyUsername'] = self.proxy_username
if self.proxy_password is not None:
kwargs['proxyPassword'] = self.proxy_password
server = Server('C://browsermob-proxy//bin//browsermob-proxy.bat', options={"port": self.server_port})
server.start()
proxy = server.create_proxy(params=kwargs)
return server, proxy
def start_proxy(self):
self.proxy_server = Server(config.proxy_bin)
self.proxy_server.start()
self.proxy = self.proxy_server.create_proxy()
if config.blacklist:
self.set_blacklist(config.blacklist)
self.proxy.new_har(self.test_id)
logger.debug('Browsermob proxy started.')
return self
def setUp(self):
options = {}
if self.browsermob_port:
options['port'] = self.browsermob_port
if not self.browsermob_script:
raise ValueError('Must specify --browsermob-script in order to '
'run browsermobproxy tests')
self.browsermob_server = Server(
self.browsermob_script, options=options)
self.browsermob_server.start()
class CreateHar(object):
"""create HTTP archive file"""
def __init__(self, mob_path):
"""initial setup"""
self.browser_mob = mob_path
self.server = self.driver = self.proxy = None
@staticmethod
def __store_into_file(title, result):
"""store result"""
har_file = open(title + '.har', 'w')
har_file.write(str(result))
har_file.close()
def __start_server(self):
"""prepare and start server"""
self.server = Server(self.browser_mob)
self.server.start()
self.proxy = self.server.create_proxy()
def __start_driver(self):
"""prepare and start driver"""
profile = webdriver.FirefoxProfile()
profile.set_proxy(self.proxy.selenium_proxy())
self.driver = webdriver.Firefox(firefox_profile=profile)
def start_all(self):
"""start server and driver"""
self.__start_server()
self.__start_driver()
def create_har(self, title, url):
"""start request and parse response"""
self.proxy.new_har(title)
self.driver.get(url)
result = json.dumps(self.proxy.har, ensure_ascii=False)
self.__store_into_file(title, result)
def stop_all(self):
"""stop server and driver"""
self.server.stop()
self.driver.quit()
def get_driver_and_proxy():
global display
global driver
global proxy
if not driver:
if int(config.get('browsermob', {}).get('collect-har', 0)):
from browsermobproxy import Server
server = Server(config['browsermob']['path'])
server.start()
proxy = server.create_proxy()
if int(config.get('xconfig', {}).get('headless', 0)):
display = Display(visible=0, size=(800, 600))
display.start()
profile = webdriver.FirefoxProfile()
if proxy:
profile.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(firefox_profile=profile)
driver.implicitly_wait(60)
return driver, proxy
def bmp_proxy():
"""
Creates a proxy and a server instance with browsermobproxy.
Reference: http://browsermob-proxy-py.readthedocs.org/en/latest/index.html
Returns:
(proxy, server)
"""
def create_proxy():
"""
Try to create a proxy.
"""
try:
proxy = server.create_proxy()
except:
return False, None
return True, proxy
port = int(os.environ.get('BROWSERMOB_PROXY_PORT', 8080))
server = Server('browsermob-proxy', options={'port': port})
try:
# If anything in this block raises an exception, make sure we kill
# the server process before exiting.
server.start()
# Using the promise module to wait for the server to be responsive.
# The server.create_proxy function sometimes raises connection
# refused errors if the server isn't ready yet.
proxy = Promise(
create_proxy, 'browsermobproxy is responsive', timeout=10
).fulfill()
proxy_host = os.environ.get('BROWSERMOB_PROXY_HOST', '127.0.0.1')
proxy.remap_hosts('localhost', proxy_host)
except:
# Make sure that the server process is stopped.
stop_server(server)
raise
return proxy, server
class Proxy(object):
proxy = None
proxy_server = None
test_id = None
def __init__(self, test_id):
self.test_id = test_id
self.start_proxy()
def start_proxy(self):
self.proxy_server = Server(config.proxy_bin)
self.proxy_server.start()
self.proxy = self.proxy_server.create_proxy()
if config.blacklist:
self.set_blacklist(config.blacklist)
self.proxy.new_har(self.test_id)
logger.debug('Browsermob proxy started.')
return self
def stop_proxy(self):
filename = '{}.har'.format(self.test_id)
with open(filename, 'w') as harfile:
json.dump(self.proxy.har, harfile)
data = json.dumps(self.proxy.har, ensure_ascii=False)
self.proxy_server.stop()
self.proxy = None
self.proxy_server = None
logger.debug('Browsermob proxy stopped. HAR created: {}'
.format(filename))
def set_blacklist(self, domain_list):
for domain in domain_list:
self.proxy.blacklist("^https?://([a-z0-9-]+[.])*{}*.*"
.format(domain), 404)
logger.debug("Proxy blacklist set.")
def get_blacklist(self):
return requests.get('{}{}/blacklist'
.format(config.proxy_api, self.proxy.port))
class ad_driver():
_driver = None
_server = None
_proxy = None
def __init__(self, path_to_batch, browser="chrome"):
""" start browsermob proxy """
self._server = Server(path_to_batch)
self._server.start()
self._proxy = self._server.create_proxy()
""" Init browser profile """
if browser is "chrome":
PROXY = "localhost:%s" % self._proxy.port # IP:PORT or HOST:PORT
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
self._driver = webdriver.Chrome(chrome_options=chrome_options)
elif browser is "ff":
profile = webdriver.FirefoxProfile()
driver = webdriver.Firefox(firefox_profile=profile, proxy=proxy)
else:
print "Please set 'browser' variable to any of the value \n 'chrome', 'ff' !"
self._driver.maximize_window()
self._driver.implicitly_wait(20)
def execute(self, test):
self._proxy.new_har(test["name"])
self._driver.get(_test_data_dir + os.sep + test['file'])
time.sleep(2)
callToTestMethod = getattr(test_steps, test["name"])
callToTestMethod(self._driver)
har = self._proxy.har
requests = har['log']['entries']
return requests
def quit(self):
self._server.stop()
self._driver.quit()
def save_web_page_stats_to_har(url, webdriver_name, save_to_file):
"""Generate the HAR archive from an URL with the Selenium webdriver
'webdriver_name', saving the HAR file to 'save_to_file'
"""
browsermob_server = Server(Config.browsermob_executable)
browsermob_server.start()
random_port = get_a_random_free_tcp_port()
proxy_conn = browsermob_server.create_proxy({"port": random_port})
driver = create_selenium_webdriver(webdriver_name, proxy_conn)
try:
proxy_conn.new_har(url, options={'captureHeaders': True})
driver.get(url)
har_json = json.dumps(proxy_conn.har, ensure_ascii=False,
indent=4, separators=(',', ': '))
# Save '.HAR' file
with io.open(save_to_file + '.har', mode='wt', buffering=1,
encoding='utf8', errors='backslashreplace',
newline=None) as output_har_f:
output_har_f.write(unicode(har_json))
# Save '.PROF' file with profiling report (timings, sizes, etc)
with io.open(save_to_file + '.prof', mode='wb', buffering=1,
newline=None) as prof_output:
report_har_dictionary(proxy_conn.har, prof_output)
finally:
proxy_conn.close()
browsermob_server.stop()
driver.quit()
def CaptureNetworkTraffic(url,server_ip,headers,file_path):
'''
This function can be used to capture network traffic from the browser. Using this function we can capture header/cookies/http calls made from the browser
url - Page url
server_ip - remap host to for specific URL
headers - this is a dictionary of the headers to be set
file_path - File in which HAR gets stored
'''
port = {'port':9090}
server = Server("G:\\browsermob\\bin\\browsermob-proxy",port) #Path to the BrowserMobProxy
server.start()
proxy = server.create_proxy()
proxy.remap_hosts("www.example.com",server_ip)
proxy.remap_hosts("www.example1.com",server_ip)
proxy.remap_hosts("www.example2.com",server_ip)
proxy.headers(headers)
profile = webdriver.FirefoxProfile()
profile.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(firefox_profile=profile)
new = {'captureHeaders':'True','captureContent':'True'}
proxy.new_har("google",new)
driver.get(url)
proxy.har # returns a HAR JSON blob
server.stop()
driver.quit()
file1 = open(file_path,'w')
json.dump(proxy.har,file1)
file1.close()
def main(argv):
init()
parser = argparse.ArgumentParser()
parser.add_argument('-u', action='store', dest='start_url', help='Set page URL', required=True)
parser.add_argument('-c', action='store', dest='cookies_file', help='JSON file with cookies', required=False)
parser.add_argument('-w', action='store', dest='webdriver_type', help='Set WebDriver type (firefox or phantomjs, firebox by default)', default="firefox", required=False)
results = parser.parse_args()
start_url = results.start_url
cookies_file = results.cookies_file
webdriver_type = results.webdriver_type
allowed_domain = urlparse(start_url).netloc
browsermobproxy_path = get_browsermobproxy_path()
options = {
'port': 9090,
}
server = Server(browsermobproxy_path,options)
server.start()
proxy = server.create_proxy()
if webdriver_type == "phantomjs":
service_args = ['--proxy=localhost:9091','--proxy-type=http',]
driver = webdriver.PhantomJS(service_args=service_args)
driver.set_window_size(1440, 1024)
else:
profile = webdriver.FirefoxProfile()
profile.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(firefox_profile=profile)
proxy.new_har('woodpycker', options={'captureHeaders': True, 'captureContent': True})
driver.get(start_url)
if not cookies_file is None:
with open(cookies_file, 'rb') as fp:
cookies = json.load(fp)
for cookie in cookies:
driver.add_cookie(cookie)
driver.refresh()
links = driver.find_elements_by_tag_name('a')
lenl = len(links)
for i in range(0,lenl):
if links[i].is_displayed():
url = links[i].get_attribute('href')
text = links[i].get_attribute('text')
if url.find(allowed_domain) != -1:
links[i].click()
print "%s Clicked on the link '%s' with HREF '%s'" % (Fore.BLUE+"*"+Fore.RESET,Style.BRIGHT+text+Style.RESET_ALL,Style.BRIGHT+url+Style.RESET_ALL)
show_status_codes(proxy.har,allowed_domain)
driver.back()
driver.refresh()
links = driver.find_elements_by_tag_name('a')
driver.quit()
server.stop()
def start_local_server(self, binpath=None):
"""Starts a local instance of BrowserMob.
Keyword Arguments:
binpath -- The full path, including the binary name to the
browsermob-proxy binary.
"""
if binpath is None:
binpath="{0}/browsermob-proxy-2.1.0-beta-4/bin/browsermob-proxy".format(getcwd())
self.server = Server(binpath)
self.server.start()
self.proxy = self.server.create_proxy()
def init(defaultPort,path):
global lock
global serverPort
global proxyIsInit
global serverIsRunning
global proxyServer
global indexCount
lock.acquire()
serverPort = int(defaultPort)
proxyIsInit = True
print "Initializing Proxy Manager - server port : set to : " + str(serverPort)
#checks to see if the server has already started
if not serverIsRunning:
proxyServer = Server(path,{'port':int(serverPort)})
proxyServer.start()
serverIsRunning = True
lock.release()
def restart_proxy_server(self):
try:
self.server.stop()
except Exception as e:
print("Failed to stop proxy server. Exception: " + str(e))
# Start Proxy
self.server = Server(self.browser_mob_proxy_location)
self.server.start()
print("Proxy server restarted")
def _setup_proxy_server(self, downstream_kbps=None, upstream_kbps=None,
latency=None):
server = Server(BROWSERMOB_PROXY_PATH)
server.start()
proxy = server.create_proxy()
# The proxy server is pretty sluggish, setting the limits might not
# achieve the desired behavior.
proxy_options = {}
if downstream_kbps:
proxy_options['downstream_kbps'] = downstream_kbps
if upstream_kbps:
proxy_options['upstream_kbps'] = upstream_kbps
if latency:
proxy_options['latency'] = latency
if len(proxy_options.items()) > 0:
proxy.limits(proxy_options)
return server, proxy
class BrowserMobProxyTestCaseMixin(object):
def __init__(self, *args, **kwargs):
self.browsermob_server = None
self.browsermob_port = kwargs.pop('browsermob_port')
self.browsermob_script = kwargs.pop('browsermob_script')
def setUp(self):
options = {}
if self.browsermob_port:
options['port'] = self.browsermob_port
if not self.browsermob_script:
raise ValueError('Must specify --browsermob-script in order to '
'run browsermobproxy tests')
self.browsermob_server = Server(
self.browsermob_script, options=options)
self.browsermob_server.start()
def create_browsermob_proxy(self):
client = self.browsermob_server.create_proxy()
with self.marionette.using_context('chrome'):
self.marionette.execute_script("""
Components.utils.import("resource://gre/modules/Preferences.jsm");
Preferences.set("network.proxy.type", 1);
Preferences.set("network.proxy.http", "localhost");
Preferences.set("network.proxy.http_port", {port});
Preferences.set("network.proxy.ssl", "localhost");
Preferences.set("network.proxy.ssl_port", {port});
""".format(port=client.port))
return client
def tearDown(self):
if self.browsermob_server:
self.browsermob_server.stop()
self.browsermob_server = None
__del__ = tearDown
def __enter__(self):
if self.virtual_display:
log.info('starting virtual display')
self.display = Display(visible=0, size=(
self.virtual_display_size_x,
self.virtual_display_size_y
))
self.display.start()
log.info('starting browsermob proxy')
self.server = Server('{}/bin/browsermob-proxy'.format(
self.browsermob_dir)
)
self.server.start()
return self
class BrowserMobProxyTestCaseMixin(object):
def __init__(self, *args, **kwargs):
self.browsermob_server = None
self.browsermob_port = kwargs.pop('browsermob_port')
self.browsermob_script = kwargs.pop('browsermob_script')
def setUp(self):
options = {}
if self.browsermob_port:
options['port'] = self.browsermob_port
if not self.browsermob_script:
raise ValueError('Must specify --browsermob-script in order to '
'run browsermobproxy tests')
self.browsermob_server = Server(
self.browsermob_script, options=options)
self.browsermob_server.start()
def create_browsermob_proxy(self):
client = self.browsermob_server.create_proxy()
with self.marionette.using_context('chrome'):
self.marionette.execute_script("""
Services.prefs.setIntPref('network.proxy.type', 1);
Services.prefs.setCharPref('network.proxy.http', 'localhost');
Services.prefs.setIntPref('network.proxy.http_port', %(port)s);
Services.prefs.setCharPref('network.proxy.ssl', 'localhost');
Services.prefs.setIntPref('network.proxy.ssl_port', %(port)s);
""" % {"port": client.port})
return client
def tearDown(self):
if self.browsermob_server:
self.browsermob_server.stop()
self.browsermob_server = None
__del__ = tearDown
def __init__(self, chromedriverPath, browsermobPath, harfilePath, cookies=None):
self.harfilePath = harfilePath
self.server = Server(browsermobPath)
self.server.start()
self.proxy = self.server.create_proxy()
os.environ["webdriver.chrome.driver"] = chromedriverPath
url = urlparse (self.proxy.proxy).path
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--proxy-server={0}".format(url))
self.driver = webdriver.Chrome(chromedriverPath,chrome_options =chrome_options)
if cookies:
print("Loading cookies from "+str(cookies))
with open(cookies, 'r') as cookieFile:
cookieJson = json.loads(cookieFile.read())
for cookie in cookieJson:
self.driver.add_cookie(cookie)
def __init__(self, path_to_batch, browser="chrome"):
""" start browsermob proxy """
self._server = Server(path_to_batch)
self._server.start()
self._proxy = self._server.create_proxy()
""" Init browser profile """
if browser is "chrome":
PROXY = "localhost:%s" % self._proxy.port # IP:PORT or HOST:PORT
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
self._driver = webdriver.Chrome(chrome_options=chrome_options)
elif browser is "ff":
profile = webdriver.FirefoxProfile()
driver = webdriver.Firefox(firefox_profile=profile, proxy=proxy)
else:
print "Please set 'browser' variable to any of the value \n 'chrome', 'ff' !"
self._driver.maximize_window()
self._driver.implicitly_wait(20)
def main():
init()
if len(sys.argv) >= 2:
start_url = sys.argv[1]
else:
print "You must specify page URL!"
sys.exit()
allowed_domain = urlparse(start_url).netloc
browsermobproxy_path = "/usr/local/opt/browsermobproxy/bin/browsermob-proxy"
options = {
'port': 9090,
}
server = Server(browsermobproxy_path,options)
server.start()
proxy = server.create_proxy()
profile = webdriver.FirefoxProfile()
profile.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(firefox_profile=profile)
driver.get(start_url)
links = driver.find_elements_by_tag_name('a')
lenl = len(links)
for i in range(0,lenl):
if links[i].is_displayed():
url = links[i].get_attribute('href')
text = links[i].get_attribute('text')
if url.find(allowed_domain) != -1:
proxy.new_har('demo')
links[i].click()
print "%s Clicked on the link '%s' with HREF '%s'" % (Fore.BLUE+"*"+Fore.RESET,Style.BRIGHT+text+Style.RESET_ALL,Style.BRIGHT+url+Style.RESET_ALL)
show_status_codes(proxy.har,allowed_domain)
driver.back()
driver.refresh()
links = driver.find_elements_by_tag_name('a')
driver.quit()
server.stop()
def fetch(url, config, output_directory, fetchEngine="browsermobproxy+selenium", browser="firefox"):
if fetchEngine in ("phantomjs", "ph"):
data = subprocess.check_output( config['fetchEngines']['phantomjs_command'].replace("$url", url), shell=True )
elif fetchEngine in ("browsermobproxy+selenium", "bs"):
from browsermobproxy import Server
from selenium import webdriver
server = Server(config['fetchEngines']['browsermobproxy_binary'])
server.start()
proxy = server.create_proxy()
if browser in ("firefox", "ff"):
profile = webdriver.FirefoxProfile()
profile.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(firefox_profile=profile)
else:
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--proxy-server={0}".format(proxy.proxy))
driver = webdriver.Chrome(chrome_options = chrome_options)
proxy.new_har(url, options={'captureHeaders': True})
driver.get(url)
data = json.dumps(proxy.har, ensure_ascii=False)
server.stop()
driver.quit()
else:
sys.exit("Unrecognized engine.")
if (data):
fileName = output_directory + "/" + url.replace("http://", "").replace("https://", "") + "_" + strftime("%Y-%m-%d_%H:%M:%S", gmtime()) + ".har"
f = open(fileName, "w")
f.write(data.encode("utf8"))
f.close()
return fileName
else:
return None
def create_hars(urls, browsermob_dir, run_cached):
for url in urls:
print 'starting browsermob proxy'
server = Server('{}/bin/browsermob-proxy'.format(browsermob_dir))
server.start()
proxy = server.create_proxy()
profile = webdriver.FirefoxProfile()
profile.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(firefox_profile=profile)
url_slug = slugify(url)
proxy.new_har(url_slug)
print 'loading page: {}'.format(url)
driver.get(url)
har_name = '{}-{}.har'.format(url_slug, time.time())
print 'saving HAR file: {}'.format(har_name)
save_har(har_name, proxy.har)
if run_cached:
url_slug = '{}-cached'.format(slugify(url))
proxy.new_har(url_slug)
print 'loading cached page: {}'.format(url)
driver.get(url)
har_name = '{}-{}.har'.format(url_slug, time.time())
print 'saving HAR file: {}'.format(har_name)
save_har(har_name, proxy.har)
driver.quit()
print 'stopping browsermob proxy'
server.stop()
from browsermobproxy import Server
server = Server("/root/Desktop/browsermob-proxy-2.1.0-beta-4/bin/browsermob-proxy")
server.start()
proxy = server.create_proxy()
from selenium import webdriver
profile = webdriver.FirefoxProfile()
profile.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(firefox_profile=profile)
proxy.new_har("google")
driver.get("http://www.google.co.uk")
test = proxy.har # returns a HAR JSON blob
print test
server.stop()
driver.quit()
class HarProfiler:
def __init__(self, config, url, login_first=False):
self.url = url
self.login_first = login_first
self.login_user = config.get('login_user')
self.login_password = config.get('login_password')
self.browsermob_dir = config['browsermob_dir']
self.har_dir = config['har_dir']
self.label_prefix = config['label_prefix'] or ''
self.run_cached = config['run_cached']
self.virtual_display = config['virtual_display']
self.virtual_display_size_x = config['virtual_display_size_x']
self.virtual_display_size_y = config['virtual_display_size_y']
self.label = '{}{}'.format(self.label_prefix, self.slugify(url))
self.cached_label = '{}-cached'.format(self.label)
epoch = time.time()
self.har_name = '{}-{}.har'.format(self.label, epoch)
self.cached_har_name = '{}-{}.har'.format(self.cached_label, epoch)
def __enter__(self):
if self.virtual_display:
log.info('starting virtual display')
self.display = Display(visible=0, size=(
self.virtual_display_size_x,
self.virtual_display_size_y
))
self.display.start()
log.info('starting browsermob proxy')
self.server = Server('{}/bin/browsermob-proxy'.format(
self.browsermob_dir)
)
self.server.start()
return self
def __exit__(self, type, value, traceback):
log.info('stopping browsermob proxy')
self.server.stop()
if self.virtual_display:
log.info('stopping virtual display')
self.display.stop()
def _make_proxied_webdriver(self):
proxy = self.server.create_proxy()
profile = webdriver.FirefoxProfile()
profile.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(firefox_profile=profile)
return (driver, proxy)
def _save_har(self, har, cached=False):
if not os.path.isdir(self.har_dir):
os.makedirs(self.har_dir)
if not cached:
har_name = self.har_name
elif cached:
har_name = self.cached_har_name
log.info('saving HAR file: {}'.format(har_name))
with open(os.path.join(self.har_dir, har_name), 'w') as f:
json.dump(har, f, indent=2, ensure_ascii=False)
def _login(self, driver):
log.info('logging in...')
error_msg = 'must specify login credentials in yaml config file'
if self.login_user is None:
raise RuntimeError(error_msg)
if self.login_password is None:
raise RuntimeError(error_msg)
driver.get('https://courses.edx.org/login')
# handle both old and new style logins
try:
email_field = driver.find_element_by_id('email')
password_field = driver.find_element_by_id('password')
except NoSuchElementException:
email_field = driver.find_element_by_id('login-email')
password_field = driver.find_element_by_id('login-password')
email_field.send_keys(self.login_user)
password_field.send_keys(self.login_password)
password_field.submit()
def _add_page_event_timings(self, driver, har):
jscript = textwrap.dedent("""
var performance = window.performance || {};
var timings = performance.timing || {};
return timings;
""")
timings = driver.execute_script(jscript)
har['log']['pages'][0]['pageTimings']['onContentLoad'] = (
timings['domContentLoadedEventEnd'] - timings['navigationStart']
)
har['log']['pages'][0]['pageTimings']['onLoad'] = (
timings['loadEventEnd'] - timings['navigationStart']
)
return har
def load_page(self):
try:
driver, proxy = self._make_proxied_webdriver()
if self.login_first:
self._login(driver)
proxy.new_har(self.label)
log.info('loading page: {}'.format(self.url))
driver.get(self.url)
har = self._add_page_event_timings(driver, proxy.har)
self._save_har(har)
if self.run_cached:
proxy.new_har(self.cached_label)
log.info('loading cached page: {}'.format(self.url))
driver.get(self.url)
har = self._add_page_event_timings(driver, proxy.har)
self._save_har(har, cached=True)
except Exception:
raise
finally:
driver.quit()
def slugify(self, text):
pattern = re.compile(r'[^a-z0-9]+')
slug = '-'.join(word for word in pattern.split(text.lower()) if word)
return slug
class HarProfiler:
def __init__(self, config, url):
self.har_dir = config['har_dir']
self.browsermob_dir = config['browsermob_dir']
self.label_prefix = config['label_prefix'] or ''
self.virtual_display = config['virtual_display']
self.virtual_display_size_x = config['virtual_display_size_x']
self.virtual_display_size_y = config['virtual_display_size_y']
self.label = '{}{}'.format(
self.label_prefix,
format(self.slugify(url))
)
self.cached_label = '{}-cached'.format(self.label)
epoch = time.time()
self.har_name = '{}-{}.har'.format(self.label, epoch)
self.cached_har_name = '{}-{}.har'.format(self.cached_label, epoch)
def __enter__(self):
log.info('starting virtual display')
if self.virtual_display:
self.display = Display(visible=0, size=(
self.virtual_display_size_x,
self.virtual_display_size_y
))
self.display.start()
log.info('starting browsermob proxy')
self.server = Server('{}/bin/browsermob-proxy'.format(
self.browsermob_dir)
)
self.server.start()
return self
def __exit__(self, type, value, traceback):
log.info('stopping browsermob proxy')
self.server.stop()
log.info('stopping virtual display')
self.display.stop()
def _make_proxied_webdriver(self):
proxy = self.server.create_proxy()
profile = webdriver.FirefoxProfile()
profile.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(firefox_profile=profile)
return (driver, proxy)
def _save_har(self, har, cached=False):
if not os.path.isdir(self.har_dir):
os.makedirs(self.har_dir)
if not cached:
har_name = self.har_name
elif cached:
har_name = self.cached_har_name
log.info('saving HAR file: {}'.format(har_name))
with open(os.path.join(self.har_dir, har_name), 'w' ) as f:
json.dump(har, f, indent=2, ensure_ascii=False)
def load_page(self, url, run_cached=True):
driver, proxy = self._make_proxied_webdriver()
proxy.new_har(self.label)
log.info('loading page: {}'.format(url))
driver.get(url)
self._save_har(proxy.har)
if run_cached:
proxy.new_har(self.cached_label)
log.info('loading cached page: {}'.format(url))
driver.get(url)
self._save_har(proxy.har, cached=True)
driver.quit()
def slugify(self, text):
pattern = re.compile(r'[^a-z0-9]+')
slug = '-'.join(word for word in pattern.split(text.lower()) if word)
return slug
