def run_webdriver(self, start_url, port, config, download_dir): """ Run Selenium WebDriver """ useragent = None referer = None webdriver = None urllib3_logger = logging.getLogger('urllib3') urllib3_logger.setLevel(logging.DEBUG) self.logger.info("Starting WebRunner") firefox_profile = None server = None proxy = None har = None if config.referer: referer = config.referer else: referer = 'http://www.google.com/search?q={}+&oq={}&oe=utf-8&rls=org.mozilla:en-US:official&client=firefox-a&channel=fflb&gws_rd=cr'.format( config.url, config.url) if config.useragent: useragent = config.useragent else: useragent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:24.0) Gecko/20100101 Firefox/24.0' self.logger.debug("Running with UserAgent: {}".format(useragent)) self.logger.debug("Running with Referer: {}".format(referer)) self.logger.debug("Checking URL: {}".format(config.url)) server = Server("lib/browsermob/bin/browsermob-proxy", {'port': port}) server.start() proxy = server.create_proxy() proxy.headers({'User-Agent': useragent, 'Accept-Encoding': "", 'Connection': 'Close'}) request_js = ( 'var referer = request.getProxyRequest().getField("Referer");' 'addReferer(request);' 'function addReferer(r){' 'if (! referer ) {' 'r.addRequestHeader("Referer","' + referer + '");' '}' 'return;' '}') proxy.request_interceptor(request_js) from types import NoneType if config.firefoxprofile is not None and os.path.isdir(config.firefoxprofile): self.logger.debug("Using existing firefox profile") firefox_profile = FirefoxProfile(profile_directory=config.firefoxprofile) else: firefox_profile = FirefoxProfile() self.logger.debug("Using profile {}".format(firefox_profile.path)) firefox_profile.set_preference("security.OCSP.enabled", 0) firefox_profile.set_preference("browser.download.folderList", 2) firefox_profile.set_preference("browser.download.manager.showWhenStarting", False) firefox_profile.set_preference("browser.download.dir", download_dir) firefox_profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/x-xpinstall;application/x-zip;application/x-zip-compressed;application/octet-stream;application/zip;application/pdf;application/msword;text/plain;application/octet") firefox_profile.set_preference("browser.helperApps.alwaysAsk.force", False) firefox_profile.set_preference("browser.download.manager.showWhenStarting", False) firefox_profile.set_preference("security.mixed_content.block_active_content", False) firefox_profile.set_preference("security.mixed_content.block_display_content", False) firefox_profile.set_preference("extensions.blocklist.enabled", False) firefox_profile.set_preference("network.proxy.type", 1) firefox_profile.set_proxy(proxy.selenium_proxy()) firefox_profile.set_preference("webdriver.log.file", "/tmp/ff.log") firefox_profile.set_preference("webdriver.log.driver", "DEBUG") firefox_profile.set_preference("browser.newtabpage.enhanced", False) firefox_profile.set_preference("browser.newtabpage.enabled", False) firefox_profile.set_preference("browser.newtabpage.directory.ping", "") firefox_profile.set_preference("browser.newtabpage.directory.source", "") firefox_profile.set_preference("browser.search.geoip.url", "") try: self.xvfb.start() capabilities = DesiredCapabilities.FIREFOX capabilities['loggingPrefs'] = {'browser':'ALL'} if os.path.exists("{}/firefox".format(firefox_profile.path)): binary = FirefoxBinary("{}/firefox".format(firefox_profile.path)) else: binary = FirefoxBinary("/usr/bin/firefox") webdriver = WebDriver(capabilities=capabilities, firefox_profile=firefox_profile, firefox_binary=binary) proxy.new_har(start_url.hostname, options={"captureHeaders": "true", "captureContent": "true", "captureBinaryContent": "true"}) self.analyse_page(webdriver, start_url) for entry in webdriver.get_log('browser'): self.logger.info("Firefox: {}".format(entry)) har = proxy.har self.logger.info("Stopping WebRunner") proxy.close() server.stop() webdriver.quit() har = Har(har) except Exception, e: self.logger.error(e) proxy.close() if webdriver: webdriver.quit() self.xvfb.stop() server.stop()