def init_browser(self): # response listen proxy server = Server(r'C:\python_job\khedu-test\browsermob-proxy-2.1.4\bin\browsermob-proxy.bat') server.start() proxy = server.create_proxy() proxy.new_har(options={ 'captureContent': True, 'captureHeaders': True }) # -- chromedriver -- chrome_options = Options() chrome_options.add_argument('--ignore-certificate-errors') chrome_options.add_argument('--proxy-server={0}'.format(proxy.proxy)) chrome_options.add_argument('--headless') # 無頭模式 browser = webdriver.Chrome(r'C:\python_job\khedu-test\chromedriver.exe', options=chrome_options) browser.maximize_window() time.sleep(1)#變大視窗需要等待 browser.set_page_load_timeout(30) wait = WebDriverWait(browser,30) browser.get(login_url) # -- login -- id0 = pq(browser.page_source)('div.z-page').attr('id').replace('_', '') # 每次重load網站都有個隨機id browser.find_element_by_xpath('//*[@id="{}b"]'.format(id0)).send_keys(username) browser.find_element_by_xpath('//*[@id="{}c"]'.format(id0)).send_keys(password) browser.find_element_by_xpath('//*[@id="{}g"]'.format(id0)).click() time.sleep(2) # not so fast browser.refresh() #頁面會卡在轉圈圈 需要reload time.sleep(2) return browser, proxy, wait
def initializeChrome(): global driver global server global proxy dict = {'port': 8090} path_to_bmp = Path( "./binaries/browsermob-proxy-2.1.4/bin/browsermob-proxy").absolute() path_on_windows = str(PureWindowsPath(path_to_bmp)) server = Server(path=path_on_windows, options=dict) #server = Server(path="binaries/browsermob-proxy-2.1.4/bin/browsermob-proxy", options=dict) server.start() proxy = server.create_proxy() user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.50 Safari/537.36' username_windows = input( "Please enter your username the way it's written in C:/Users/ - it's needed to locate your Chrome user data." ) user_data_path = Path('C:/Users/' + str(username_windows) + '/AppData/Local/Google/Chrome/User Data') chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--disable-extensions") chrome_options.add_argument("--proxy-server={0}".format(proxy.proxy)) chrome_options.add_experimental_option("excludeSwitches", ['enable-automation']) chrome_options.add_argument(f'--user-data-dir={user_data_path}') path_to_chromedriver = str(Path("./binaries/chromedriver.exe").absolute()) driver = webdriver.Chrome(path_to_chromedriver, options=chrome_options) print('initialized Chrome window!')
def setUp(self): """ Start the browser with a browsermob-proxy instance for use by the test. You *must* call this in the `setUp` method of any subclasses before using the browser! Returns: None """ try: # Start server proxy server = Server('browsermob-proxy') server.start() self.proxy = server.create_proxy() proxy_host = os.environ.get('BROWSERMOB_PROXY_HOST', '127.0.0.1') self.proxy.remap_hosts('localhost', proxy_host) except: self.skipTest('Skipping: could not start server with browsermob-proxy.') # parent's setUp super(WebAppPerfReport, self).setUp() # Initialize vars self._page_timings = [] self._active_har = False self._with_cache = False # Add one more cleanup for the server self.addCleanup(server.stop)
def __start_bmproxy(self, config): # BrowserMob from browsermobproxy import Server from arjuna.tpi.constant import ArjunaOption capture_traffic = config.value(ArjunaOption.BROWSER_NETWORK_RECORDER_ENABLED) if capture_traffic: bmproxy_dir = config.value(ArjunaOption.TOOLS_BMPROXY_DIR) sub_dirs = os.listdir(bmproxy_dir) bmproxy_bin_dir = None if "bin" in sub_dirs: bmproxy_bin_dir = os.path.join(bmproxy_dir, "bin") else: sub_dirs.sort(reverse=True) # Last version will be picked. for d in sub_dir: if d.startswith("browsermob"): bmproxy_bin_dir = os.path.join(bmproxy_dir, d, "bin") break if bmproxy_bin_dir is None: raise Exception("Network recording is enabled in configuration. There was an error in creating proxy server/server using BrowserMob Proxy. Could not find proxy package at {}".format(bmproxy_dir)) if platform.system().lower() == "windows": exe = "browsermob-proxy.bat" else: exe = "browsermob-proxy" bmproxy_exe_path = os.path.join(bmproxy_bin_dir, exe) try: self.__bmproxy_server = Server(bmproxy_exe_path) self.__bmproxy_server.start() except ProxyServerError as e: raise Exception("Network recording is enabled in configuration. There was an error in creating proxy server/server using BrowserMob Proxy. Fix and retry. Error message: {}".format(str(e)))
class testAuthBasic(unittest.TestCase): def setUp(self): self.server = Server( r"C:\Users\user\Documents\ScrinShots\SeleniumLessons\basic_auth\browsermob-proxy\bin\browsermob-proxy" ) self.server.start() self.proxy = self.server.create_proxy() self.profile = webdriver.FirefoxProfile() self.profile.set_proxy(self.proxy.selenium_proxy()) self.driver = webdriver.Firefox(firefox_profile=self.profile) self.driver.implicitly_wait(8) self.driver.get("https://auth-demo.aerobatic.io/") def tearDown(self): self.server.stop() self.driver.quit() def test_search(self): self.proxy.basic_authentication("auth-demo.aerobatic.io", "aerobatic", "aerobatic") self.driver.get("https://auth-demo.aerobatic.io/protected-standard/") assert self.driver.find_element_by_css_selector( "a.button-primary[href='/']")
class GetWebPageHar: proxy_path = "./browsermob-proxy-2.1.4/bin/browsermob-proxy" def __init__(self): self.server = Server(str(Path(self.proxy_path).absolute())) self.server.start() self.proxy = self.server.create_proxy() def get_har(self, url, output_filename): chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--proxy-server={0}".format( self.proxy.proxy)) chrome_options.add_argument("--ignore-certificate-errors") chrome_options.add_argument("--incognito") driver = webdriver.Chrome(chrome_options=chrome_options) self.proxy.new_har("google") driver.get(url) time.sleep(10) har_json = json.dumps(self.proxy.har, indent=4, ensure_ascii=False) with open(output_filename, "w") as f: f.write(har_json) driver.quit() def stop(self): self.server.stop()
def __init__(self, results, reports, **kwargs): self.results = results self.reports = reports self.args = kwargs self.listen_port = 9760 self.lock = threading.Lock() self.vulnerable = [] self.server = None self.cookies = {} for entry in self.args['cookie'].split(';'): if entry.find('=') == -1: continue key, value = entry.strip().split('=', 1) self.cookies[key] = value # Create proxy server logging.info('Starting browsermobproxy server...') self.proxy_server = Server(self.args['browsermobproxy']) self.proxy_server.start() self.proxy = self.proxy_server.create_proxy() logging.info('Browsermobproxy server started') # Create Chrome engine logging.info('Creating Selenium Chrome webdriver...') self.chrome_options = webdriver.ChromeOptions() self.chrome_options.add_argument('--proxy-server={}'.format( self.proxy.proxy)) if 'headless' in self.args: self.chrome_options.add_argument('--headless') self.chrome_options.add_argument('--disable-gpu') self.chrome_options.add_argument("--disable-extensions") self.driver = webdriver.Chrome(chrome_options=self.chrome_options) logging.info('Selenium Chrome webdriver created')
def reloadHeaderAndCookie(): global httpArchive browsermob_path = '/usr/local/browsermob-proxy-2.1.4/bin/browsermob-proxy' server = Server(browsermob_path) proxy = server.create_proxy() chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--proxy-server={0}".format(proxy.proxy)) browser = webdriver.Chrome(options=chrome_options) options = {'captureHeaders': True, 'captureCookies': True} signIn(browser) proxy.new_har("lifemiles", options=options) url = "https://www.lifemiles.com/fly/find" time.sleep(2) browser.get(url) time.sleep(5) originID = "ar_bkr_fromairport" browser.find_element_by_id(originID).send_keys("SFO") time.sleep(2) destinationID = "ar_bkr_toairport" browser.find_element_by_id(destinationID).send_keys("SFO") time.sleep(2) server.start() searchClass = "Booker_bookerActionButtonSmall__3Fh2d" try: browser.find_element_by_class_name(searchClass).click() except Exception, e: print("was not able to click search button, try javascript") browser.execute_script( "document.querySelector('.{0}').click()".format(searchClass))
class ProxyDistributor: browsermob_proxy_path = "/home/amerigo/PycharmProjects/Progetto/tesi/risorse/browsermob-proxy/bin/browsermob-proxy" def __init__(self): self.server = Server(self.browsermob_proxy_path) self.actual_port = 9090 def get_new_proxy(self): proxy = self.server.create_proxy({ 'port': self.actual_port, 'captureHeaders': True, 'captureContent': True, 'captureBinaryContent': True }) self.actual_port += 1 return proxy def start(self): if self.server is None: self.server = Server(self.browsermob_proxy_path) self.server.start() def stop(self): if self.server is not None: self.server.stop()
def __init__(self, url, username, password): self.bearer = None # enable browser logging d = DesiredCapabilities.CHROME d['loggingPrefs'] = {'browser': 'ALL'} self.server = Server("BrowserMob\\bin\\browsermob-proxy") cli.print_info("Starting proxy server...") self.server.start() self.proxy = self.server.create_proxy() cli.add_to_print("OK\n\n") cli.print_warning("NOTE: Connections will show as INSECURE!\n") options = webdriver.ChromeOptions() self.url = url options.add_argument("--proxy-server={0}".format(self.proxy.proxy)) cli.print_info("Browser started\n") self.browser = Chrome(options=options, executable_path="./chromedriver.exe", desired_capabilities=d) self.username = username self.password = password del password del username
def setupdevices(): """ Description: Sets u browser proxy, Selenium driver, and har object Usage: [driver,proxy]=setupdevices() Inputs: NA Output: Selenium driver Browsermob proxy Browsermob server """ #set up proxy server = Server("############/browsermob-proxy-2.0-beta-9/bin/browsermob-proxy") server.start() proxy = server.create_proxy() profile = webdriver.FirefoxProfile() profile.set_proxy(proxy.selenium_proxy()) proxy.new_har("________") #set up driver driver = webdriver.Firefox(firefox_profile=profile) return (driver,proxy,server)
def create_server(browser, websites): print("=====> Creating New Server - Please Wait... <=====") server = Server(browsermob_proxy_path) server.start() proxy = server.create_proxy() configure_server(proxy, browser, websites) close_server(proxy, server)
class ProxyManager: __BMP = filename def __init__(self): self.__server = Server(ProxyManager.__BMP) self.__client = None def start_server(self): self.__server.start() print('start proxy server') return self.__server def start_client(self): self.__client = self.__server.create_proxy( params={"trustAllServers": "true"}) print('start proxy client') return self.__client @property def client(self): return self.__client @property def server(self): return self.__server
def test2(): browsermob_path = '/usr/local/browsermob-proxy-2.1.4/bin/browsermob-proxy' server = Server(browsermob_path, {'port':9999}) time.sleep(1) proxy = server.create_proxy() time.sleep(1)
def get_driver(self): # server = Server(r'/home/reggie/Documents/app/browsermob-proxy-2.1.4/bin/browsermob-proxy') server = Server(r'/root/browsermob-proxy-2.1.4/bin/browsermob-proxy') server.start() print('server started') # map={'http':'http://officepx.datayes.com','https':'http://officepx.datayes.com'} proxy = server.create_proxy() print('proxy created') caps = DesiredCapabilities().CHROME caps["pageLoadStrategy"] = "none" chrome_options = Options() chrome_options.add_argument('--ignore-ssl-errors=yes') chrome_options.add_argument('--ignore-certificate-errors') prefs = {'profile.managed_default_content_settings.images': 2} chrome_options.add_experimental_option('prefs', prefs) chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--headless') chrome_options.add_argument('--disable-gpu') chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.add_experimental_option('excludeSwitches', ['enable-automation']) # 以开发者模式 chrome_options.add_argument('--proxy-server={0}'.format(proxy.proxy)) drive = webdriver.Chrome(options=chrome_options, desired_capabilities=caps) print('driver opened') return drive, server, proxy
def get_driver(request, cmdopt_browser, cmdopt_window): """Fixture to create, return and close driver""" server = Server( "C:\\Program Files (x86)\\browsermob-proxy\\bin\\browsermob-proxy", {"port": 9090}) server.start() proxy = server.create_proxy() url = urllib.parse.urlparse(proxy.proxy).path driver = None if cmdopt_browser == "ie": driver = webdriver.Ie() elif cmdopt_browser == "firefox": if cmdopt_window == "headless": options = Firefox_options() options.add_argument("--headless") options.add_argument('--proxy-server={}'.format(url)) driver = webdriver.Firefox(firefox_options=options) else: options = Firefox_options() options.add_argument('--proxy-server={}'.format(url)) driver = webdriver.Firefox() proxy.new_har() request.addfinalizer(driver.quit) elif cmdopt_browser == "chrome": if cmdopt_window == "headless": options = Chrome_options() options.headless = True options.add_argument('--proxy-server={}'.format(url)) driver = webdriver.Chrome(chrome_options=options) proxy.new_har() request.addfinalizer(driver.quit) else: chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--proxy-server={}'.format(url)) d = DesiredCapabilities.CHROME d['loggingPrefs'] = {'browser': 'ALL'} driver = webdriver.Chrome(desired_capabilities=d, chrome_options=chrome_options) proxy.new_har() ef_driver = EventFiringWebDriver(driver, MyListener()) def fin(): log_timestamp = str(datetime.datetime.now())[0:-4].replace( '-', '.').replace(' ', '_').replace(':', '.') browserlog_filename = log_timestamp + '_browser_log_file.log' browserlogfile = open(browserlog_filename, 'w') print('-------------------------') for i in ef_driver.get_log('browser'): print(i) browserlogfile.write(str(i) + '\n') print(proxy.har) server.stop() ef_driver.quit request.addfinalizer(fin) return ef_driver else: return "unsupported browser" return driver
class ProxyManager: __BMP = './browsermob-proxy-2.1.4/bin/browsermob-proxy' def __init__(self): self.__server = Server(ProxyManager.__BMP) self.__client = None def start_server(self): self.__client = self.__server.create_proxy( params={'trustAllServers': 'true'}) self.__server.start() return self.__server def start_client(self): return self.__client @property def client(self): return self.__client @property def server(self): return self.__server
class BrowserMobProxyTestCaseMixin(object): def __init__(self, *args, **kwargs): self.browsermob_server = None self.browsermob_port = kwargs.pop('browsermob_port') self.browsermob_script = kwargs.pop('browsermob_script') def setUp(self): options = {} if self.browsermob_port: options['port'] = self.browsermob_port if not self.browsermob_script: raise ValueError('Must specify --browsermob-script in order to ' 'run browsermobproxy tests') self.browsermob_server = Server(self.browsermob_script, options=options) self.browsermob_server.start() def create_browsermob_proxy(self): client = self.browsermob_server.create_proxy() with self.marionette.using_context('chrome'): self.marionette.execute_script(""" Services.prefs.setIntPref('network.proxy.type', 1); Services.prefs.setCharPref('network.proxy.http', 'localhost'); Services.prefs.setIntPref('network.proxy.http_port', %(port)s); Services.prefs.setCharPref('network.proxy.ssl', 'localhost'); Services.prefs.setIntPref('network.proxy.ssl_port', %(port)s); """ % {"port": client.port}) return client def tearDown(self): if self.browsermob_server: self.browsermob_server.stop() self.browsermob_server = None __del__ = tearDown
class BaseFramework(object): def __init__(self): self.server = Server('./browsermob-proxy-2.1.4/bin/browsermob-proxy') self.server.start() self.proxy = self.server.create_proxy() chrome_options = Options() chrome_options.add_argument('--ignore-certificate-errors') chrome_options.add_argument('--proxy-server={0}'.format(self.proxy.proxy)) self.browser = webdriver.Chrome(options=chrome_options) def process_request(self, request, response): pass def process_response(self, response, request): pass def run(self, func, *args): self.proxy.new_har(options={ 'captureContent': True, 'captureHeaders': True }) func(*args) result = self.proxy.har for entry in result['log']['entries']: request = entry['request'] response = entry['response'] self.process_request(request, response) self.process_response(response, request) def __del__(self): self.proxy.close() self.browser.close()
def __init__(self, url, har_name, browsermob_proxy_location, selector_dictionary=None, default_timeout=None, firefox_binary=None, highlight=False, geckodriver="geckodriver"): self.selector_dictionary = selector_dictionary self.default_timeout = default_timeout if default_timeout is not None else 30 self.highlight = highlight for proc in psutil.process_iter(): # check whether the process name matches if proc.name() == "browsermob-proxy": proc.kill() options = {'port': 8090} # self.server = Server(path="../tools/browsermob-proxy-2.1.4/bin/browsermob-proxy", options=dict) self.server = Server(path=browsermob_proxy_location, options=options) self.server.start() time.sleep(1) self.proxy = self.server.create_proxy() time.sleep(1) profile = webdriver.FirefoxProfile() selenium_proxy = self.proxy.selenium_proxy() profile.set_proxy(selenium_proxy) self.driver = webdriver.Firefox(firefox_profile=profile, firefox_binary=firefox_binary, executable_path=geckodriver) self.proxy.new_har(har_name) self.driver.get(url)
def login(weburl, user, passwd, proxy_location, webdriver_location): server = Server(proxy_location + r'\browsermob-proxy.bat') server.start() proxy = server.create_proxy() proxy.new_har(options={'captureContent': True, 'captureHeaders': True}) chrome_options = Options() chrome_options.add_argument('--ignore-certificate-errors') chrome_options.add_argument('--proxy-server={}'.format(proxy.proxy)) browser = webdriver.Chrome(webdriver_location + r'\chromedriver', options=chrome_options) browser.set_page_load_timeout(30) wait = WebDriverWait(browser, 10) #登入 browser.get(login_url) id0 = pq(browser.page_source)('div.z-page').attr('id').replace( '_', '') #抓第一行 每個id,都會隨機亂產生 browser.find_element_by_xpath('//*[@id="{}b"]'.format(id0)).send_keys( username) #亂數+b browser.find_element_by_xpath('//*[@id="{}c"]'.format(id0)).send_keys( password) #亂數+c browser.find_element_by_xpath('//*[@id="{}g"]'.format(id0)).click() #亂數+g time.sleep(4) content_id = pq(browser.page_source)('div.z-page').attr('id').replace( '_', '') #主頁面的亂數 print(id0) print(content_id) #回傳 browser, wait ,webid, proxy return browser, wait, proxy, content_id
def reloadHeaderAndCookie(): browsermob_path = '/usr/local/browsermob-proxy-2.1.4/bin/browsermob-proxy' server = Server(browsermob_path, {'port': 9999}) time.sleep(1) proxy = server.create_proxy() time.sleep(1)
def __init__(self, root="C:/xampp/htdocs/webscrape/", folder=""): self.url = "" self.root = root + folder + "/" # Start browsermob proxy self.server = Server( r"C:\webdrivers\browsermob-proxy\bin\browsermob-proxy") self.server.start() self.proxy = self.server.create_proxy() # Setup Chrome webdriver - note: does not seem to work with headless On options = webdriver.ChromeOptions() options.binary_location = r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe" options.add_argument( '--proxy-server=%s' % self.proxy.proxy ) # Setup proxy to point to our browsermob so that it can track requests self.w = webdriver.Chrome(r'C:/webdrivers/chromedriver.exe', chrome_options=options) self.proxy.new_har("Listener", options={'captureHeaders': True}) # Request listener #self.proxy.new_har("Listener" ) # Request listener print("Browser and Server initialized...")
def _start_ProxyHelper(self, options, proxy_port=None): """ 启动代理类,用于监听Http请求 :param options: chrom浏览器启动选项 :param proxy_port: 代理端口号 """ assert self._proxyHelper is None, "代理已开启" # 代理端口,若未指定,获取默认端口 proxy_port = GF.proxy_port() if proxy_port is None else proxy_port # 启动代理浏览器 server = Server(GF.proxy_path(), {'port': proxy_port}) server.start({'log_path': GF.cache_path()}) proxy = server.create_proxy() # 开启har监听 proxy.new_har(options={'captureContent': True, 'captureHeaders': True}) options.add_argument('--proxy-server={0}'.format(proxy.proxy)) # 启动代理日志 self.init_proxy_log() # 启动代理助手 self._proxyHelper = ProxyHelper(self, proxy) return options
def __init__(self): self.server = Server('./browsermob-proxy-2.1.4/bin/browsermob-proxy') self.server.start() self.proxy = self.server.create_proxy() chrome_options = Options() chrome_options.add_argument('--ignore-certificate-errors') chrome_options.add_argument('--proxy-server={0}'.format(self.proxy.proxy)) self.browser = webdriver.Chrome(options=chrome_options)
def get_driver(self, browser, start_beacon_url): server = Server(BROWSERMOB_LOCATION) server.start() self.proxy = server.create_proxy() driver = webdriver.Firefox(proxy=self.proxy.selenium_proxy()) self.proxy.new_har() self.beacon_url = start_beacon_url return driver
def __init__(self, headless: bool = True): super().__init__(headless=headless) ProxyFirefoxRunner._force_shutdown_of_browsermob_proxy() sleep(1) self.proxy_server = Server(BROWSERMOB_PROXY) self.proxy_server.start() sleep(1) self.proxy_client = self.proxy_server.create_proxy() sleep(1)
def test(): browsermob_path = '/usr/local/browsermob-proxy-2.1.4/bin/browsermob-proxy' server = Server(browsermob_path, {'port': 8090}) server.start() time.sleep(1) proxy = server.create_proxy() time.sleep(1) print("we started proxy")
def get_proxy_and_server(passed_dict): print('Getting Browsermob proxy and server') server = Server('qa/env/bin/browsermob-proxy/bin/browsermob-proxy', options={'existing_proxy_port_to_use': 8999}) server.start() proxy = server.create_proxy() proxy.headers({"Authorization": passed_dict["Authorization"]}) proxy.new_har(HOST) return proxy, server
def proxy(): """Фикстура для конфигурирования прокси-сервера""" server = Server( "03_PageObject/browsermob-proxy-2.1.4/bin/browsermob-proxy") server.start() client = Client("localhost:8080") # print(client.port) client.new_har() return server, client
def __init__(self, path=None): if path is None: self.server = Server(config['HAR']['SERVER_PATH']) self.server.start() self.proxy = self._start_proxy() self.driver = self._start_chrome_driver() self.df = None else: self.df = pd.read_csv(f'{path}')
def initProxy(self): """ step 4 初始化 browermobproxy 设置需要屏蔽的网络连接,此处屏蔽了css,和图片(有时chrome的设置会失效),可加快网页加载速度 新建proxy代理地址 """ self.server = Server(self.PROXY_PATH) self.server.start() self.proxy = self.server.create_proxy()
def pytest_runtest_setup(item): logger = logging.getLogger(__name__) if item.config.option.bmp_test_proxy and 'skip_browsermob_proxy' not in item.keywords: if hasattr(item.session.config, 'browsermob_server'): server = item.session.config.browsermob_server else: server = Server(item.config.option.bmp_path, {'port': int(item.config.option.bmp_port)}) item.config.browsermob_test_proxy = server.create_proxy() logger.info('BrowserMob test proxy started (%s:%s)' % (item.config.option.bmp_host, item.config.browsermob_test_proxy.port)) configure_browsermob_proxy(item.config.browsermob_test_proxy, item.config) #TODO make recording of har configurable item.config.browsermob_test_proxy.new_har()
def init_proxy_server(self, port=None): kwargs = {} if port is not None: kwargs['port'] = port if self.chained_proxy is not None: if self.is_https: kwargs['httpsProxy'] = self.chained_proxy else: kwargs['httpProxy'] = self.chained_proxy if self.proxy_username is not None: kwargs['proxyUsername'] = self.proxy_username if self.proxy_password is not None: kwargs['proxyPassword'] = self.proxy_password server = Server('C://browsermob-proxy//bin//browsermob-proxy.bat', options={"port": self.server_port}) server.start() proxy = server.create_proxy(params=kwargs) return server, proxy
def start_proxy(self): self.proxy_server = Server(config.proxy_bin) self.proxy_server.start() self.proxy = self.proxy_server.create_proxy() if config.blacklist: self.set_blacklist(config.blacklist) self.proxy.new_har(self.test_id) logger.debug('Browsermob proxy started.') return self
def setUp(self): options = {} if self.browsermob_port: options['port'] = self.browsermob_port if not self.browsermob_script: raise ValueError('Must specify --browsermob-script in order to ' 'run browsermobproxy tests') self.browsermob_server = Server( self.browsermob_script, options=options) self.browsermob_server.start()
class CreateHar(object): """create HTTP archive file""" def __init__(self, mob_path): """initial setup""" self.browser_mob = mob_path self.server = self.driver = self.proxy = None @staticmethod def __store_into_file(title, result): """store result""" har_file = open(title + '.har', 'w') har_file.write(str(result)) har_file.close() def __start_server(self): """prepare and start server""" self.server = Server(self.browser_mob) self.server.start() self.proxy = self.server.create_proxy() def __start_driver(self): """prepare and start driver""" profile = webdriver.FirefoxProfile() profile.set_proxy(self.proxy.selenium_proxy()) self.driver = webdriver.Firefox(firefox_profile=profile) def start_all(self): """start server and driver""" self.__start_server() self.__start_driver() def create_har(self, title, url): """start request and parse response""" self.proxy.new_har(title) self.driver.get(url) result = json.dumps(self.proxy.har, ensure_ascii=False) self.__store_into_file(title, result) def stop_all(self): """stop server and driver""" self.server.stop() self.driver.quit()
def get_driver_and_proxy(): global display global driver global proxy if not driver: if int(config.get('browsermob', {}).get('collect-har', 0)): from browsermobproxy import Server server = Server(config['browsermob']['path']) server.start() proxy = server.create_proxy() if int(config.get('xconfig', {}).get('headless', 0)): display = Display(visible=0, size=(800, 600)) display.start() profile = webdriver.FirefoxProfile() if proxy: profile.set_proxy(proxy.selenium_proxy()) driver = webdriver.Firefox(firefox_profile=profile) driver.implicitly_wait(60) return driver, proxy
def bmp_proxy(): """ Creates a proxy and a server instance with browsermobproxy. Reference: http://browsermob-proxy-py.readthedocs.org/en/latest/index.html Returns: (proxy, server) """ def create_proxy(): """ Try to create a proxy. """ try: proxy = server.create_proxy() except: return False, None return True, proxy port = int(os.environ.get('BROWSERMOB_PROXY_PORT', 8080)) server = Server('browsermob-proxy', options={'port': port}) try: # If anything in this block raises an exception, make sure we kill # the server process before exiting. server.start() # Using the promise module to wait for the server to be responsive. # The server.create_proxy function sometimes raises connection # refused errors if the server isn't ready yet. proxy = Promise( create_proxy, 'browsermobproxy is responsive', timeout=10 ).fulfill() proxy_host = os.environ.get('BROWSERMOB_PROXY_HOST', '127.0.0.1') proxy.remap_hosts('localhost', proxy_host) except: # Make sure that the server process is stopped. stop_server(server) raise return proxy, server
class Proxy(object): proxy = None proxy_server = None test_id = None def __init__(self, test_id): self.test_id = test_id self.start_proxy() def start_proxy(self): self.proxy_server = Server(config.proxy_bin) self.proxy_server.start() self.proxy = self.proxy_server.create_proxy() if config.blacklist: self.set_blacklist(config.blacklist) self.proxy.new_har(self.test_id) logger.debug('Browsermob proxy started.') return self def stop_proxy(self): filename = '{}.har'.format(self.test_id) with open(filename, 'w') as harfile: json.dump(self.proxy.har, harfile) data = json.dumps(self.proxy.har, ensure_ascii=False) self.proxy_server.stop() self.proxy = None self.proxy_server = None logger.debug('Browsermob proxy stopped. HAR created: {}' .format(filename)) def set_blacklist(self, domain_list): for domain in domain_list: self.proxy.blacklist("^https?://([a-z0-9-]+[.])*{}*.*" .format(domain), 404) logger.debug("Proxy blacklist set.") def get_blacklist(self): return requests.get('{}{}/blacklist' .format(config.proxy_api, self.proxy.port))
class ad_driver(): _driver = None _server = None _proxy = None def __init__(self, path_to_batch, browser="chrome"): """ start browsermob proxy """ self._server = Server(path_to_batch) self._server.start() self._proxy = self._server.create_proxy() """ Init browser profile """ if browser is "chrome": PROXY = "localhost:%s" % self._proxy.port # IP:PORT or HOST:PORT chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--proxy-server=%s' % PROXY) self._driver = webdriver.Chrome(chrome_options=chrome_options) elif browser is "ff": profile = webdriver.FirefoxProfile() driver = webdriver.Firefox(firefox_profile=profile, proxy=proxy) else: print "Please set 'browser' variable to any of the value \n 'chrome', 'ff' !" self._driver.maximize_window() self._driver.implicitly_wait(20) def execute(self, test): self._proxy.new_har(test["name"]) self._driver.get(_test_data_dir + os.sep + test['file']) time.sleep(2) callToTestMethod = getattr(test_steps, test["name"]) callToTestMethod(self._driver) har = self._proxy.har requests = har['log']['entries'] return requests def quit(self): self._server.stop() self._driver.quit()
def save_web_page_stats_to_har(url, webdriver_name, save_to_file): """Generate the HAR archive from an URL with the Selenium webdriver 'webdriver_name', saving the HAR file to 'save_to_file' """ browsermob_server = Server(Config.browsermob_executable) browsermob_server.start() random_port = get_a_random_free_tcp_port() proxy_conn = browsermob_server.create_proxy({"port": random_port}) driver = create_selenium_webdriver(webdriver_name, proxy_conn) try: proxy_conn.new_har(url, options={'captureHeaders': True}) driver.get(url) har_json = json.dumps(proxy_conn.har, ensure_ascii=False, indent=4, separators=(',', ': ')) # Save '.HAR' file with io.open(save_to_file + '.har', mode='wt', buffering=1, encoding='utf8', errors='backslashreplace', newline=None) as output_har_f: output_har_f.write(unicode(har_json)) # Save '.PROF' file with profiling report (timings, sizes, etc) with io.open(save_to_file + '.prof', mode='wb', buffering=1, newline=None) as prof_output: report_har_dictionary(proxy_conn.har, prof_output) finally: proxy_conn.close() browsermob_server.stop() driver.quit()
def CaptureNetworkTraffic(url,server_ip,headers,file_path): ''' This function can be used to capture network traffic from the browser. Using this function we can capture header/cookies/http calls made from the browser url - Page url server_ip - remap host to for specific URL headers - this is a dictionary of the headers to be set file_path - File in which HAR gets stored ''' port = {'port':9090} server = Server("G:\\browsermob\\bin\\browsermob-proxy",port) #Path to the BrowserMobProxy server.start() proxy = server.create_proxy() proxy.remap_hosts("www.example.com",server_ip) proxy.remap_hosts("www.example1.com",server_ip) proxy.remap_hosts("www.example2.com",server_ip) proxy.headers(headers) profile = webdriver.FirefoxProfile() profile.set_proxy(proxy.selenium_proxy()) driver = webdriver.Firefox(firefox_profile=profile) new = {'captureHeaders':'True','captureContent':'True'} proxy.new_har("google",new) driver.get(url) proxy.har # returns a HAR JSON blob server.stop() driver.quit() file1 = open(file_path,'w') json.dump(proxy.har,file1) file1.close()
def main(argv): init() parser = argparse.ArgumentParser() parser.add_argument('-u', action='store', dest='start_url', help='Set page URL', required=True) parser.add_argument('-c', action='store', dest='cookies_file', help='JSON file with cookies', required=False) parser.add_argument('-w', action='store', dest='webdriver_type', help='Set WebDriver type (firefox or phantomjs, firebox by default)', default="firefox", required=False) results = parser.parse_args() start_url = results.start_url cookies_file = results.cookies_file webdriver_type = results.webdriver_type allowed_domain = urlparse(start_url).netloc browsermobproxy_path = get_browsermobproxy_path() options = { 'port': 9090, } server = Server(browsermobproxy_path,options) server.start() proxy = server.create_proxy() if webdriver_type == "phantomjs": service_args = ['--proxy=localhost:9091','--proxy-type=http',] driver = webdriver.PhantomJS(service_args=service_args) driver.set_window_size(1440, 1024) else: profile = webdriver.FirefoxProfile() profile.set_proxy(proxy.selenium_proxy()) driver = webdriver.Firefox(firefox_profile=profile) proxy.new_har('woodpycker', options={'captureHeaders': True, 'captureContent': True}) driver.get(start_url) if not cookies_file is None: with open(cookies_file, 'rb') as fp: cookies = json.load(fp) for cookie in cookies: driver.add_cookie(cookie) driver.refresh() links = driver.find_elements_by_tag_name('a') lenl = len(links) for i in range(0,lenl): if links[i].is_displayed(): url = links[i].get_attribute('href') text = links[i].get_attribute('text') if url.find(allowed_domain) != -1: links[i].click() print "%s Clicked on the link '%s' with HREF '%s'" % (Fore.BLUE+"*"+Fore.RESET,Style.BRIGHT+text+Style.RESET_ALL,Style.BRIGHT+url+Style.RESET_ALL) show_status_codes(proxy.har,allowed_domain) driver.back() driver.refresh() links = driver.find_elements_by_tag_name('a') driver.quit() server.stop()
def start_local_server(self, binpath=None): """Starts a local instance of BrowserMob. Keyword Arguments: binpath -- The full path, including the binary name to the browsermob-proxy binary. """ if binpath is None: binpath="{0}/browsermob-proxy-2.1.0-beta-4/bin/browsermob-proxy".format(getcwd()) self.server = Server(binpath) self.server.start() self.proxy = self.server.create_proxy()
def init(defaultPort,path): global lock global serverPort global proxyIsInit global serverIsRunning global proxyServer global indexCount lock.acquire() serverPort = int(defaultPort) proxyIsInit = True print "Initializing Proxy Manager - server port : set to : " + str(serverPort) #checks to see if the server has already started if not serverIsRunning: proxyServer = Server(path,{'port':int(serverPort)}) proxyServer.start() serverIsRunning = True lock.release()
def restart_proxy_server(self): try: self.server.stop() except Exception as e: print("Failed to stop proxy server. Exception: " + str(e)) # Start Proxy self.server = Server(self.browser_mob_proxy_location) self.server.start() print("Proxy server restarted")
def _setup_proxy_server(self, downstream_kbps=None, upstream_kbps=None, latency=None): server = Server(BROWSERMOB_PROXY_PATH) server.start() proxy = server.create_proxy() # The proxy server is pretty sluggish, setting the limits might not # achieve the desired behavior. proxy_options = {} if downstream_kbps: proxy_options['downstream_kbps'] = downstream_kbps if upstream_kbps: proxy_options['upstream_kbps'] = upstream_kbps if latency: proxy_options['latency'] = latency if len(proxy_options.items()) > 0: proxy.limits(proxy_options) return server, proxy
class BrowserMobProxyTestCaseMixin(object): def __init__(self, *args, **kwargs): self.browsermob_server = None self.browsermob_port = kwargs.pop('browsermob_port') self.browsermob_script = kwargs.pop('browsermob_script') def setUp(self): options = {} if self.browsermob_port: options['port'] = self.browsermob_port if not self.browsermob_script: raise ValueError('Must specify --browsermob-script in order to ' 'run browsermobproxy tests') self.browsermob_server = Server( self.browsermob_script, options=options) self.browsermob_server.start() def create_browsermob_proxy(self): client = self.browsermob_server.create_proxy() with self.marionette.using_context('chrome'): self.marionette.execute_script(""" Components.utils.import("resource://gre/modules/Preferences.jsm"); Preferences.set("network.proxy.type", 1); Preferences.set("network.proxy.http", "localhost"); Preferences.set("network.proxy.http_port", {port}); Preferences.set("network.proxy.ssl", "localhost"); Preferences.set("network.proxy.ssl_port", {port}); """.format(port=client.port)) return client def tearDown(self): if self.browsermob_server: self.browsermob_server.stop() self.browsermob_server = None __del__ = tearDown
def __enter__(self): if self.virtual_display: log.info('starting virtual display') self.display = Display(visible=0, size=( self.virtual_display_size_x, self.virtual_display_size_y )) self.display.start() log.info('starting browsermob proxy') self.server = Server('{}/bin/browsermob-proxy'.format( self.browsermob_dir) ) self.server.start() return self
class BrowserMobProxyTestCaseMixin(object): def __init__(self, *args, **kwargs): self.browsermob_server = None self.browsermob_port = kwargs.pop('browsermob_port') self.browsermob_script = kwargs.pop('browsermob_script') def setUp(self): options = {} if self.browsermob_port: options['port'] = self.browsermob_port if not self.browsermob_script: raise ValueError('Must specify --browsermob-script in order to ' 'run browsermobproxy tests') self.browsermob_server = Server( self.browsermob_script, options=options) self.browsermob_server.start() def create_browsermob_proxy(self): client = self.browsermob_server.create_proxy() with self.marionette.using_context('chrome'): self.marionette.execute_script(""" Services.prefs.setIntPref('network.proxy.type', 1); Services.prefs.setCharPref('network.proxy.http', 'localhost'); Services.prefs.setIntPref('network.proxy.http_port', %(port)s); Services.prefs.setCharPref('network.proxy.ssl', 'localhost'); Services.prefs.setIntPref('network.proxy.ssl_port', %(port)s); """ % {"port": client.port}) return client def tearDown(self): if self.browsermob_server: self.browsermob_server.stop() self.browsermob_server = None __del__ = tearDown
def __init__(self, chromedriverPath, browsermobPath, harfilePath, cookies=None): self.harfilePath = harfilePath self.server = Server(browsermobPath) self.server.start() self.proxy = self.server.create_proxy() os.environ["webdriver.chrome.driver"] = chromedriverPath url = urlparse (self.proxy.proxy).path chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--proxy-server={0}".format(url)) self.driver = webdriver.Chrome(chromedriverPath,chrome_options =chrome_options) if cookies: print("Loading cookies from "+str(cookies)) with open(cookies, 'r') as cookieFile: cookieJson = json.loads(cookieFile.read()) for cookie in cookieJson: self.driver.add_cookie(cookie)
def __init__(self, path_to_batch, browser="chrome"): """ start browsermob proxy """ self._server = Server(path_to_batch) self._server.start() self._proxy = self._server.create_proxy() """ Init browser profile """ if browser is "chrome": PROXY = "localhost:%s" % self._proxy.port # IP:PORT or HOST:PORT chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--proxy-server=%s' % PROXY) self._driver = webdriver.Chrome(chrome_options=chrome_options) elif browser is "ff": profile = webdriver.FirefoxProfile() driver = webdriver.Firefox(firefox_profile=profile, proxy=proxy) else: print "Please set 'browser' variable to any of the value \n 'chrome', 'ff' !" self._driver.maximize_window() self._driver.implicitly_wait(20)
def main(): init() if len(sys.argv) >= 2: start_url = sys.argv[1] else: print "You must specify page URL!" sys.exit() allowed_domain = urlparse(start_url).netloc browsermobproxy_path = "/usr/local/opt/browsermobproxy/bin/browsermob-proxy" options = { 'port': 9090, } server = Server(browsermobproxy_path,options) server.start() proxy = server.create_proxy() profile = webdriver.FirefoxProfile() profile.set_proxy(proxy.selenium_proxy()) driver = webdriver.Firefox(firefox_profile=profile) driver.get(start_url) links = driver.find_elements_by_tag_name('a') lenl = len(links) for i in range(0,lenl): if links[i].is_displayed(): url = links[i].get_attribute('href') text = links[i].get_attribute('text') if url.find(allowed_domain) != -1: proxy.new_har('demo') links[i].click() print "%s Clicked on the link '%s' with HREF '%s'" % (Fore.BLUE+"*"+Fore.RESET,Style.BRIGHT+text+Style.RESET_ALL,Style.BRIGHT+url+Style.RESET_ALL) show_status_codes(proxy.har,allowed_domain) driver.back() driver.refresh() links = driver.find_elements_by_tag_name('a') driver.quit() server.stop()
def fetch(url, config, output_directory, fetchEngine="browsermobproxy+selenium", browser="firefox"): if fetchEngine in ("phantomjs", "ph"): data = subprocess.check_output( config['fetchEngines']['phantomjs_command'].replace("$url", url), shell=True ) elif fetchEngine in ("browsermobproxy+selenium", "bs"): from browsermobproxy import Server from selenium import webdriver server = Server(config['fetchEngines']['browsermobproxy_binary']) server.start() proxy = server.create_proxy() if browser in ("firefox", "ff"): profile = webdriver.FirefoxProfile() profile.set_proxy(proxy.selenium_proxy()) driver = webdriver.Firefox(firefox_profile=profile) else: chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--proxy-server={0}".format(proxy.proxy)) driver = webdriver.Chrome(chrome_options = chrome_options) proxy.new_har(url, options={'captureHeaders': True}) driver.get(url) data = json.dumps(proxy.har, ensure_ascii=False) server.stop() driver.quit() else: sys.exit("Unrecognized engine.") if (data): fileName = output_directory + "/" + url.replace("http://", "").replace("https://", "") + "_" + strftime("%Y-%m-%d_%H:%M:%S", gmtime()) + ".har" f = open(fileName, "w") f.write(data.encode("utf8")) f.close() return fileName else: return None
def create_hars(urls, browsermob_dir, run_cached): for url in urls: print 'starting browsermob proxy' server = Server('{}/bin/browsermob-proxy'.format(browsermob_dir)) server.start() proxy = server.create_proxy() profile = webdriver.FirefoxProfile() profile.set_proxy(proxy.selenium_proxy()) driver = webdriver.Firefox(firefox_profile=profile) url_slug = slugify(url) proxy.new_har(url_slug) print 'loading page: {}'.format(url) driver.get(url) har_name = '{}-{}.har'.format(url_slug, time.time()) print 'saving HAR file: {}'.format(har_name) save_har(har_name, proxy.har) if run_cached: url_slug = '{}-cached'.format(slugify(url)) proxy.new_har(url_slug) print 'loading cached page: {}'.format(url) driver.get(url) har_name = '{}-{}.har'.format(url_slug, time.time()) print 'saving HAR file: {}'.format(har_name) save_har(har_name, proxy.har) driver.quit() print 'stopping browsermob proxy' server.stop()
from browsermobproxy import Server server = Server("/root/Desktop/browsermob-proxy-2.1.0-beta-4/bin/browsermob-proxy") server.start() proxy = server.create_proxy() from selenium import webdriver profile = webdriver.FirefoxProfile() profile.set_proxy(proxy.selenium_proxy()) driver = webdriver.Firefox(firefox_profile=profile) proxy.new_har("google") driver.get("http://www.google.co.uk") test = proxy.har # returns a HAR JSON blob print test server.stop() driver.quit()
class HarProfiler: def __init__(self, config, url, login_first=False): self.url = url self.login_first = login_first self.login_user = config.get('login_user') self.login_password = config.get('login_password') self.browsermob_dir = config['browsermob_dir'] self.har_dir = config['har_dir'] self.label_prefix = config['label_prefix'] or '' self.run_cached = config['run_cached'] self.virtual_display = config['virtual_display'] self.virtual_display_size_x = config['virtual_display_size_x'] self.virtual_display_size_y = config['virtual_display_size_y'] self.label = '{}{}'.format(self.label_prefix, self.slugify(url)) self.cached_label = '{}-cached'.format(self.label) epoch = time.time() self.har_name = '{}-{}.har'.format(self.label, epoch) self.cached_har_name = '{}-{}.har'.format(self.cached_label, epoch) def __enter__(self): if self.virtual_display: log.info('starting virtual display') self.display = Display(visible=0, size=( self.virtual_display_size_x, self.virtual_display_size_y )) self.display.start() log.info('starting browsermob proxy') self.server = Server('{}/bin/browsermob-proxy'.format( self.browsermob_dir) ) self.server.start() return self def __exit__(self, type, value, traceback): log.info('stopping browsermob proxy') self.server.stop() if self.virtual_display: log.info('stopping virtual display') self.display.stop() def _make_proxied_webdriver(self): proxy = self.server.create_proxy() profile = webdriver.FirefoxProfile() profile.set_proxy(proxy.selenium_proxy()) driver = webdriver.Firefox(firefox_profile=profile) return (driver, proxy) def _save_har(self, har, cached=False): if not os.path.isdir(self.har_dir): os.makedirs(self.har_dir) if not cached: har_name = self.har_name elif cached: har_name = self.cached_har_name log.info('saving HAR file: {}'.format(har_name)) with open(os.path.join(self.har_dir, har_name), 'w') as f: json.dump(har, f, indent=2, ensure_ascii=False) def _login(self, driver): log.info('logging in...') error_msg = 'must specify login credentials in yaml config file' if self.login_user is None: raise RuntimeError(error_msg) if self.login_password is None: raise RuntimeError(error_msg) driver.get('https://courses.edx.org/login') # handle both old and new style logins try: email_field = driver.find_element_by_id('email') password_field = driver.find_element_by_id('password') except NoSuchElementException: email_field = driver.find_element_by_id('login-email') password_field = driver.find_element_by_id('login-password') email_field.send_keys(self.login_user) password_field.send_keys(self.login_password) password_field.submit() def _add_page_event_timings(self, driver, har): jscript = textwrap.dedent(""" var performance = window.performance || {}; var timings = performance.timing || {}; return timings; """) timings = driver.execute_script(jscript) har['log']['pages'][0]['pageTimings']['onContentLoad'] = ( timings['domContentLoadedEventEnd'] - timings['navigationStart'] ) har['log']['pages'][0]['pageTimings']['onLoad'] = ( timings['loadEventEnd'] - timings['navigationStart'] ) return har def load_page(self): try: driver, proxy = self._make_proxied_webdriver() if self.login_first: self._login(driver) proxy.new_har(self.label) log.info('loading page: {}'.format(self.url)) driver.get(self.url) har = self._add_page_event_timings(driver, proxy.har) self._save_har(har) if self.run_cached: proxy.new_har(self.cached_label) log.info('loading cached page: {}'.format(self.url)) driver.get(self.url) har = self._add_page_event_timings(driver, proxy.har) self._save_har(har, cached=True) except Exception: raise finally: driver.quit() def slugify(self, text): pattern = re.compile(r'[^a-z0-9]+') slug = '-'.join(word for word in pattern.split(text.lower()) if word) return slug
class HarProfiler: def __init__(self, config, url): self.har_dir = config['har_dir'] self.browsermob_dir = config['browsermob_dir'] self.label_prefix = config['label_prefix'] or '' self.virtual_display = config['virtual_display'] self.virtual_display_size_x = config['virtual_display_size_x'] self.virtual_display_size_y = config['virtual_display_size_y'] self.label = '{}{}'.format( self.label_prefix, format(self.slugify(url)) ) self.cached_label = '{}-cached'.format(self.label) epoch = time.time() self.har_name = '{}-{}.har'.format(self.label, epoch) self.cached_har_name = '{}-{}.har'.format(self.cached_label, epoch) def __enter__(self): log.info('starting virtual display') if self.virtual_display: self.display = Display(visible=0, size=( self.virtual_display_size_x, self.virtual_display_size_y )) self.display.start() log.info('starting browsermob proxy') self.server = Server('{}/bin/browsermob-proxy'.format( self.browsermob_dir) ) self.server.start() return self def __exit__(self, type, value, traceback): log.info('stopping browsermob proxy') self.server.stop() log.info('stopping virtual display') self.display.stop() def _make_proxied_webdriver(self): proxy = self.server.create_proxy() profile = webdriver.FirefoxProfile() profile.set_proxy(proxy.selenium_proxy()) driver = webdriver.Firefox(firefox_profile=profile) return (driver, proxy) def _save_har(self, har, cached=False): if not os.path.isdir(self.har_dir): os.makedirs(self.har_dir) if not cached: har_name = self.har_name elif cached: har_name = self.cached_har_name log.info('saving HAR file: {}'.format(har_name)) with open(os.path.join(self.har_dir, har_name), 'w' ) as f: json.dump(har, f, indent=2, ensure_ascii=False) def load_page(self, url, run_cached=True): driver, proxy = self._make_proxied_webdriver() proxy.new_har(self.label) log.info('loading page: {}'.format(url)) driver.get(url) self._save_har(proxy.har) if run_cached: proxy.new_har(self.cached_label) log.info('loading cached page: {}'.format(url)) driver.get(url) self._save_har(proxy.har, cached=True) driver.quit() def slugify(self, text): pattern = re.compile(r'[^a-z0-9]+') slug = '-'.join(word for word in pattern.split(text.lower()) if word) return slug