def run_webdriver(self, start_url, port, config, download_dir):
"""
Run Selenium WebDriver
"""
useragent = None
referer = None
webdriver = None
urllib3_logger = logging.getLogger('urllib3')
urllib3_logger.setLevel(logging.DEBUG)
self.logger.info("Starting WebRunner")
firefox_profile = None
server = None
proxy = None
har = None
if config.referer:
referer = config.referer
else:
referer = 'http://www.google.com/search?q={}+&oq={}&oe=utf-8&rls=org.mozilla:en-US:official&client=firefox-a&channel=fflb&gws_rd=cr'.format(
config.url, config.url)
if config.useragent:
useragent = config.useragent
else:
useragent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:24.0) Gecko/20100101 Firefox/24.0'
self.logger.debug("Running with UserAgent: {}".format(useragent))
self.logger.debug("Running with Referer: {}".format(referer))
self.logger.debug("Checking URL: {}".format(config.url))
server = Server("lib/browsermob/bin/browsermob-proxy", {'port': port})
server.start()
proxy = server.create_proxy()
proxy.headers({'User-Agent': useragent, 'Accept-Encoding': "", 'Connection': 'Close'})
request_js = (
'var referer = request.getProxyRequest().getField("Referer");'
'addReferer(request);'
'function addReferer(r){'
'if (! referer ) {'
'r.addRequestHeader("Referer","' + referer + '");'
'}'
'return;'
'}')
proxy.request_interceptor(request_js)
from types import NoneType
if config.firefoxprofile is not None and os.path.isdir(config.firefoxprofile):
self.logger.debug("Using existing firefox profile")
firefox_profile = FirefoxProfile(profile_directory=config.firefoxprofile)
else:
firefox_profile = FirefoxProfile()
self.logger.debug("Using profile {}".format(firefox_profile.path))
firefox_profile.set_preference("security.OCSP.enabled", 0)
firefox_profile.set_preference("browser.download.folderList", 2)
firefox_profile.set_preference("browser.download.manager.showWhenStarting", False)
firefox_profile.set_preference("browser.download.dir", download_dir)
firefox_profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
"application/x-xpinstall;application/x-zip;application/x-zip-compressed;application/octet-stream;application/zip;application/pdf;application/msword;text/plain;application/octet")
firefox_profile.set_preference("browser.helperApps.alwaysAsk.force", False)
firefox_profile.set_preference("browser.download.manager.showWhenStarting", False)
firefox_profile.set_preference("security.mixed_content.block_active_content", False)
firefox_profile.set_preference("security.mixed_content.block_display_content", False)
firefox_profile.set_preference("extensions.blocklist.enabled", False)
firefox_profile.set_preference("network.proxy.type", 1)
firefox_profile.set_proxy(proxy.selenium_proxy())
firefox_profile.set_preference("webdriver.log.file", "/tmp/ff.log")
firefox_profile.set_preference("webdriver.log.driver", "DEBUG")
firefox_profile.set_preference("browser.newtabpage.enhanced", False)
firefox_profile.set_preference("browser.newtabpage.enabled", False)
firefox_profile.set_preference("browser.newtabpage.directory.ping", "")
firefox_profile.set_preference("browser.newtabpage.directory.source", "")
firefox_profile.set_preference("browser.search.geoip.url", "")
try:
self.xvfb.start()
capabilities = DesiredCapabilities.FIREFOX
capabilities['loggingPrefs'] = {'browser':'ALL'}
if os.path.exists("{}/firefox".format(firefox_profile.path)):
binary = FirefoxBinary("{}/firefox".format(firefox_profile.path))
else:
binary = FirefoxBinary("/usr/bin/firefox")
webdriver = WebDriver(capabilities=capabilities, firefox_profile=firefox_profile, firefox_binary=binary)
proxy.new_har(start_url.hostname,
options={"captureHeaders": "true", "captureContent": "true", "captureBinaryContent": "true"})
self.analyse_page(webdriver, start_url)
for entry in webdriver.get_log('browser'):
self.logger.info("Firefox: {}".format(entry))
har = proxy.har
self.logger.info("Stopping WebRunner")
proxy.close()
server.stop()
webdriver.quit()
har = Har(har)
except Exception, e:
self.logger.error(e)
proxy.close()
if webdriver:
webdriver.quit()
self.xvfb.stop()
server.stop()