def newProxytest(proxy):
if 'TEST_FIREFOX' in os.environ:
p = FirefoxProfile('/home/creature/.mozilla/firefox/s1tpg123.default')
p.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(p)
else:
o = Options()
o.add_argument("--proxy-server={0}".format(proxy.proxy))
driver = webdriver.Chrome(chrome_options=o)
driver.get("http://*****:*****@localhost:9000")
return driver
def refreshDriver(self):
# make sure Tor and Proxies has installed and configure
if hasattr(self, "driver"):
self.driver.close()
self.changeIP()
proxy_address = "localhost:8118"
proxy = Proxy()
proxy.socksProxy = proxy_address
profile = FirefoxProfile()
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': proxy_address,
'httpsProxy': proxy_address,
'ftpProxy': proxy_address,
'sslProxy': proxy_address,
'noProxy': ""
})
profile.set_proxy(proxy)
driver = Firefox(firefox_profile=profile)
self.driver = driver
def run_webdriver(self, start_url, port, config, download_dir):
global useragent
global referer
urllib3_logger = logging.getLogger('urllib3')
urllib3_logger.setLevel(logging.DEBUG)
logging.info("Starting WebRunner")
firefox_profile = None
server = None
proxy = None
har = None
if config.referer:
referer = config.referer
else:
referer = 'http://www.google.com/search?q={}+&oq={}&oe=utf-8&rls=org.mozilla:en-US:official&client=firefox-a&channel=fflb&gws_rd=cr'.format(
config.url, config.url)
if config.useragent:
useragent = config.useragent
else:
useragent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:24.0) Gecko/20100101 Firefox/24.0'
logging.debug("Running with UserAgent: {}".format(useragent))
logging.debug("Running with Referer: {}".format(referer))
logging.debug("Checking URL: {}".format(config.url))
server = Server("lib/browsermob/bin/browsermob-proxy", {'port': port})
server.start()
proxy = server.create_proxy()
proxy.headers({'User-Agent': useragent, 'Accept-Encoding': "", 'Connection': 'Close'})
request_js = (
'var referer = request.getProxyRequest().getField("Referer");'
'addReferer(request);'
'function addReferer(r){'
'if (! referer ) {'
'r.addRequestHeader("Referer","' + referer + '");'
'}'
'return;'
'}')
proxy.request_interceptor(request_js)
if config.firefoxprofile:
firefox_profile = FirefoxProfile(profile_directory=config.firefoxprofile)
else:
firefox_profile = FirefoxProfile()
logging.debug("Using profile {}".format(firefox_profile.path))
firefox_profile.set_preference("security.OCSP.enabled", 0)
firefox_profile.set_preference("browser.download.folderList", 2)
firefox_profile.set_preference("browser.download.manager.showWhenStarting", False)
firefox_profile.set_preference("browser.download.dir", download_dir)
firefox_profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
"application/x-xpinstall;application/x-zip;application/x-zip-compressed;application/octet-stream;application/zip;application/pdf;appl\
ication/msword;text/plain;application/octet")
firefox_profile.set_preference("browser.helperApps.alwaysAsk.force", False)
firefox_profile.set_preference("browser.download.manager.showWhenStarting", False)
firefox_profile.set_preference("network.proxy.type", 1)
firefox_profile.set_proxy(proxy.selenium_proxy())
try:
webdriver = WebDriver(firefox_profile)
proxy.new_har(start_url.hostname,
options={"captureHeaders": "true", "captureContent": "true", "captureBinaryContent": "true"})
self.analyse_page(webdriver, start_url)
har = proxy.har
logging.info("Stopping WebRunner")
proxy.close()
server.stop()
webdriver.quit()
har = Har(har)
except Exception, e:
logging.error(e)
proxy.close()
webdriver.quit()
server.stop()
def run_webdriver(self, start_url, port, config, download_dir):
"""
Run Selenium WebDriver
"""
useragent = None
referer = None
webdriver = None
urllib3_logger = logging.getLogger('urllib3')
urllib3_logger.setLevel(logging.DEBUG)
self.logger.info("Starting WebRunner")
firefox_profile = None
server = None
proxy = None
har = None
if config.referer:
referer = config.referer
else:
referer = 'http://www.google.com/search?q={}+&oq={}&oe=utf-8&rls=org.mozilla:en-US:official&client=firefox-a&channel=fflb&gws_rd=cr'.format(
config.url, config.url)
if config.useragent:
useragent = config.useragent
else:
useragent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:24.0) Gecko/20100101 Firefox/24.0'
self.logger.debug("Running with UserAgent: {}".format(useragent))
self.logger.debug("Running with Referer: {}".format(referer))
self.logger.debug("Checking URL: {}".format(config.url))
server = Server("lib/browsermob/bin/browsermob-proxy", {'port': port})
server.start()
proxy = server.create_proxy()
proxy.headers({'User-Agent': useragent, 'Accept-Encoding': "", 'Connection': 'Close'})
request_js = (
'var referer = request.getProxyRequest().getField("Referer");'
'addReferer(request);'
'function addReferer(r){'
'if (! referer ) {'
'r.addRequestHeader("Referer","' + referer + '");'
'}'
'return;'
'}')
proxy.request_interceptor(request_js)
from types import NoneType
if config.firefoxprofile is not None and os.path.isdir(config.firefoxprofile):
self.logger.debug("Using existing firefox profile")
firefox_profile = FirefoxProfile(profile_directory=config.firefoxprofile)
else:
firefox_profile = FirefoxProfile()
self.logger.debug("Using profile {}".format(firefox_profile.path))
firefox_profile.set_preference("security.OCSP.enabled", 0)
firefox_profile.set_preference("browser.download.folderList", 2)
firefox_profile.set_preference("browser.download.manager.showWhenStarting", False)
firefox_profile.set_preference("browser.download.dir", download_dir)
firefox_profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
"application/x-xpinstall;application/x-zip;application/x-zip-compressed;application/octet-stream;application/zip;application/pdf;application/msword;text/plain;application/octet")
firefox_profile.set_preference("browser.helperApps.alwaysAsk.force", False)
firefox_profile.set_preference("browser.download.manager.showWhenStarting", False)
firefox_profile.set_preference("security.mixed_content.block_active_content", False)
firefox_profile.set_preference("security.mixed_content.block_display_content", False)
firefox_profile.set_preference("extensions.blocklist.enabled", False)
firefox_profile.set_preference("network.proxy.type", 1)
firefox_profile.set_proxy(proxy.selenium_proxy())
firefox_profile.set_preference("webdriver.log.file", "/tmp/ff.log")
firefox_profile.set_preference("webdriver.log.driver", "DEBUG")
firefox_profile.set_preference("browser.newtabpage.enhanced", False)
firefox_profile.set_preference("browser.newtabpage.enabled", False)
firefox_profile.set_preference("browser.newtabpage.directory.ping", "")
firefox_profile.set_preference("browser.newtabpage.directory.source", "")
firefox_profile.set_preference("browser.search.geoip.url", "")
try:
self.xvfb.start()
capabilities = DesiredCapabilities.FIREFOX
capabilities['loggingPrefs'] = {'browser':'ALL'}
if os.path.exists("{}/firefox".format(firefox_profile.path)):
binary = FirefoxBinary("{}/firefox".format(firefox_profile.path))
else:
binary = FirefoxBinary("/usr/bin/firefox")
webdriver = WebDriver(capabilities=capabilities, firefox_profile=firefox_profile, firefox_binary=binary)
proxy.new_har(start_url.hostname,
options={"captureHeaders": "true", "captureContent": "true", "captureBinaryContent": "true"})
self.analyse_page(webdriver, start_url)
for entry in webdriver.get_log('browser'):
self.logger.info("Firefox: {}".format(entry))
har = proxy.har
self.logger.info("Stopping WebRunner")
proxy.close()
server.stop()
webdriver.quit()
har = Har(har)
except Exception, e:
self.logger.error(e)
proxy.close()
if webdriver:
webdriver.quit()
self.xvfb.stop()
server.stop()
