def newProxytest(proxy): if 'TEST_FIREFOX' in os.environ: p = FirefoxProfile('/home/creature/.mozilla/firefox/s1tpg123.default') p.set_proxy(proxy.selenium_proxy()) driver = webdriver.Firefox(p) else: o = Options() o.add_argument("--proxy-server={0}".format(proxy.proxy)) driver = webdriver.Chrome(chrome_options=o) driver.get("http://*****:*****@localhost:9000") return driver
def refreshDriver(self): # make sure Tor and Proxies has installed and configure if hasattr(self, "driver"): self.driver.close() self.changeIP() proxy_address = "localhost:8118" proxy = Proxy() proxy.socksProxy = proxy_address profile = FirefoxProfile() proxy = Proxy({ 'proxyType': ProxyType.MANUAL, 'httpProxy': proxy_address, 'httpsProxy': proxy_address, 'ftpProxy': proxy_address, 'sslProxy': proxy_address, 'noProxy': "" }) profile.set_proxy(proxy) driver = Firefox(firefox_profile=profile) self.driver = driver
def run_webdriver(self, start_url, port, config, download_dir): global useragent global referer urllib3_logger = logging.getLogger('urllib3') urllib3_logger.setLevel(logging.DEBUG) logging.info("Starting WebRunner") firefox_profile = None server = None proxy = None har = None if config.referer: referer = config.referer else: referer = 'http://www.google.com/search?q={}+&oq={}&oe=utf-8&rls=org.mozilla:en-US:official&client=firefox-a&channel=fflb&gws_rd=cr'.format( config.url, config.url) if config.useragent: useragent = config.useragent else: useragent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:24.0) Gecko/20100101 Firefox/24.0' logging.debug("Running with UserAgent: {}".format(useragent)) logging.debug("Running with Referer: {}".format(referer)) logging.debug("Checking URL: {}".format(config.url)) server = Server("lib/browsermob/bin/browsermob-proxy", {'port': port}) server.start() proxy = server.create_proxy() proxy.headers({'User-Agent': useragent, 'Accept-Encoding': "", 'Connection': 'Close'}) request_js = ( 'var referer = request.getProxyRequest().getField("Referer");' 'addReferer(request);' 'function addReferer(r){' 'if (! referer ) {' 'r.addRequestHeader("Referer","' + referer + '");' '}' 'return;' '}') proxy.request_interceptor(request_js) if config.firefoxprofile: firefox_profile = FirefoxProfile(profile_directory=config.firefoxprofile) else: firefox_profile = FirefoxProfile() logging.debug("Using profile {}".format(firefox_profile.path)) firefox_profile.set_preference("security.OCSP.enabled", 0) firefox_profile.set_preference("browser.download.folderList", 2) firefox_profile.set_preference("browser.download.manager.showWhenStarting", False) firefox_profile.set_preference("browser.download.dir", download_dir) firefox_profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/x-xpinstall;application/x-zip;application/x-zip-compressed;application/octet-stream;application/zip;application/pdf;appl\ ication/msword;text/plain;application/octet") firefox_profile.set_preference("browser.helperApps.alwaysAsk.force", False) firefox_profile.set_preference("browser.download.manager.showWhenStarting", False) firefox_profile.set_preference("network.proxy.type", 1) firefox_profile.set_proxy(proxy.selenium_proxy()) try: webdriver = WebDriver(firefox_profile) proxy.new_har(start_url.hostname, options={"captureHeaders": "true", "captureContent": "true", "captureBinaryContent": "true"}) self.analyse_page(webdriver, start_url) har = proxy.har logging.info("Stopping WebRunner") proxy.close() server.stop() webdriver.quit() har = Har(har) except Exception, e: logging.error(e) proxy.close() webdriver.quit() server.stop()
def run_webdriver(self, start_url, port, config, download_dir): """ Run Selenium WebDriver """ useragent = None referer = None webdriver = None urllib3_logger = logging.getLogger('urllib3') urllib3_logger.setLevel(logging.DEBUG) self.logger.info("Starting WebRunner") firefox_profile = None server = None proxy = None har = None if config.referer: referer = config.referer else: referer = 'http://www.google.com/search?q={}+&oq={}&oe=utf-8&rls=org.mozilla:en-US:official&client=firefox-a&channel=fflb&gws_rd=cr'.format( config.url, config.url) if config.useragent: useragent = config.useragent else: useragent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:24.0) Gecko/20100101 Firefox/24.0' self.logger.debug("Running with UserAgent: {}".format(useragent)) self.logger.debug("Running with Referer: {}".format(referer)) self.logger.debug("Checking URL: {}".format(config.url)) server = Server("lib/browsermob/bin/browsermob-proxy", {'port': port}) server.start() proxy = server.create_proxy() proxy.headers({'User-Agent': useragent, 'Accept-Encoding': "", 'Connection': 'Close'}) request_js = ( 'var referer = request.getProxyRequest().getField("Referer");' 'addReferer(request);' 'function addReferer(r){' 'if (! referer ) {' 'r.addRequestHeader("Referer","' + referer + '");' '}' 'return;' '}') proxy.request_interceptor(request_js) from types import NoneType if config.firefoxprofile is not None and os.path.isdir(config.firefoxprofile): self.logger.debug("Using existing firefox profile") firefox_profile = FirefoxProfile(profile_directory=config.firefoxprofile) else: firefox_profile = FirefoxProfile() self.logger.debug("Using profile {}".format(firefox_profile.path)) firefox_profile.set_preference("security.OCSP.enabled", 0) firefox_profile.set_preference("browser.download.folderList", 2) firefox_profile.set_preference("browser.download.manager.showWhenStarting", False) firefox_profile.set_preference("browser.download.dir", download_dir) firefox_profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/x-xpinstall;application/x-zip;application/x-zip-compressed;application/octet-stream;application/zip;application/pdf;application/msword;text/plain;application/octet") firefox_profile.set_preference("browser.helperApps.alwaysAsk.force", False) firefox_profile.set_preference("browser.download.manager.showWhenStarting", False) firefox_profile.set_preference("security.mixed_content.block_active_content", False) firefox_profile.set_preference("security.mixed_content.block_display_content", False) firefox_profile.set_preference("extensions.blocklist.enabled", False) firefox_profile.set_preference("network.proxy.type", 1) firefox_profile.set_proxy(proxy.selenium_proxy()) firefox_profile.set_preference("webdriver.log.file", "/tmp/ff.log") firefox_profile.set_preference("webdriver.log.driver", "DEBUG") firefox_profile.set_preference("browser.newtabpage.enhanced", False) firefox_profile.set_preference("browser.newtabpage.enabled", False) firefox_profile.set_preference("browser.newtabpage.directory.ping", "") firefox_profile.set_preference("browser.newtabpage.directory.source", "") firefox_profile.set_preference("browser.search.geoip.url", "") try: self.xvfb.start() capabilities = DesiredCapabilities.FIREFOX capabilities['loggingPrefs'] = {'browser':'ALL'} if os.path.exists("{}/firefox".format(firefox_profile.path)): binary = FirefoxBinary("{}/firefox".format(firefox_profile.path)) else: binary = FirefoxBinary("/usr/bin/firefox") webdriver = WebDriver(capabilities=capabilities, firefox_profile=firefox_profile, firefox_binary=binary) proxy.new_har(start_url.hostname, options={"captureHeaders": "true", "captureContent": "true", "captureBinaryContent": "true"}) self.analyse_page(webdriver, start_url) for entry in webdriver.get_log('browser'): self.logger.info("Firefox: {}".format(entry)) har = proxy.har self.logger.info("Stopping WebRunner") proxy.close() server.stop() webdriver.quit() har = Har(har) except Exception, e: self.logger.error(e) proxy.close() if webdriver: webdriver.quit() self.xvfb.stop() server.stop()