def __init__(self): self.tags = {} tags = Registry().get('config')['spider']['tags'] tags = tags.split(",") for tag in tags: tag = tag.split("|") self.tags[tag[0]] = tag[1]
def browser_create(self, retry_counter=0): """ Create a browser """ if retry_counter >= int(Registry().get('config')['selenium'] ['browser_recreate_errors_limit']): raise Exception( "WebDriver can`t create browser. Check errors log selenium settings." ) self_num = random.randint(0, 99999) myProxy = Registry().get('proxies').get_proxy() if myProxy: proxy = Proxy({ 'proxyType': ProxyType.MANUAL, 'httpProxy': myProxy, 'ftpProxy': myProxy, 'sslProxy': myProxy, 'noProxy': '' }) self.proxy_using = True else: #print "No proxy" proxy = None self.proxy_using = False profile_path = '/tmp/wr-selenium-{0}/'.format(self_num) if os.path.exists(profile_path): shutil.rmtree(profile_path) if not os.path.exists(profile_path): os.mkdir(profile_path) profile = webdriver.FirefoxProfile(profile_path) if Registry().get('config')['selenium']['css_load'] != '1': profile.set_preference('permissions.default.stylesheet', 2) if Registry().get('config')['selenium']['images_load'] != '1': profile.set_preference('permissions.default.image', 2) if Registry().get('config')['selenium']['flash_load'] != '1': profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false') profile.set_preference("browser.startup.homepage", "about:blank") profile.set_preference("browser.formfill.enable", "false") profile.set_preference("startup.homepage_welcome_url", "about:blank") profile.set_preference("startup.homepage_welcome_url.additional", "about:blank") if myProxy and len(myProxy): agent_IP, agent_Port = myProxy.split(":") profile.set_preference("network.proxy.type", 1) profile.set_preference("network.proxy.share_proxy_settings", True) profile.set_preference("network.http.use-cache", False) profile.set_preference("network.proxy.http", agent_IP) profile.set_preference("network.proxy.http_port", int(agent_Port)) profile.set_preference('network.proxy.ssl_port', int(agent_Port)) profile.set_preference('network.proxy.ssl', agent_IP) profile.set_preference('network.proxy.socks', agent_IP) profile.set_preference('network.proxy.socks_port', int(agent_Port)) fo = open('/tmp/firefox-run-{0}.log'.format(self_num), "w") binary = FirefoxBinary( firefox_path=Registry().get('config')['selenium']['firefox_path'], log_file=fo) try: self.browser = SeleniumBrowser( profile, firefox_binary=binary, proxy=proxy, browser_wait_re=self.browser_wait_re, ) except WebDriverException as e: self.logger.ex(e) self.logger.log("Re-trying. Browser creation error: " + str(e)) shutil.rmtree(profile_path) time.sleep(5) retry_counter += 1 return self.browser_create(retry_counter) self.browser.set_page_load_timeout( Registry().get('config')['selenium']['timeout_page_load']) self.browser.implicitly_wait( Registry().get('config')['selenium']['timeout_page_load'])
def browser_create(self, retry_counter = 0): """ Create a browser """ if retry_counter >= int(Registry().get('config')['selenium']['browser_recreate_errors_limit']): raise Exception("WebDriver can`t create browser. Check errors log selenium settings.") self_num = random.randint(0, 99999) myProxy = Registry().get('proxies').get_proxy() if myProxy: proxy = Proxy({ 'proxyType': ProxyType.MANUAL, 'httpProxy': myProxy, 'ftpProxy': myProxy, 'sslProxy': myProxy, 'noProxy': '' }) self.proxy_using = True else: #print "No proxy" proxy = None self.proxy_using = False profile_path = '/tmp/wr-selenium-{0}/'.format(self_num) if os.path.exists(profile_path): shutil.rmtree(profile_path) if not os.path.exists(profile_path): os.mkdir(profile_path) profile = webdriver.FirefoxProfile(profile_path) if Registry().get('config')['selenium']['css_load'] != '1': profile.set_preference('permissions.default.stylesheet', 2) if Registry().get('config')['selenium']['images_load'] != '1': profile.set_preference('permissions.default.image', 2) if Registry().get('config')['selenium']['flash_load'] != '1': profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false') profile.set_preference("browser.startup.homepage", "about:blank") profile.set_preference("startup.homepage_welcome_url", "about:blank") profile.set_preference("startup.homepage_welcome_url.additional", "about:blank") if myProxy and len(myProxy): agent_IP, agent_Port = myProxy.split(":") profile.set_preference("network.proxy.type", 1) profile.set_preference("network.proxy.share_proxy_settings", True) profile.set_preference("network.http.use-cache", False) profile.set_preference("network.proxy.http", agent_IP) profile.set_preference("network.proxy.http_port", int(agent_Port)) profile.set_preference('network.proxy.ssl_port', int(agent_Port)) profile.set_preference('network.proxy.ssl', agent_IP) profile.set_preference('network.proxy.socks', agent_IP) profile.set_preference('network.proxy.socks_port', int(agent_Port)) fo = open('/tmp/firefox-run-{0}.log'.format(self_num), "w") binary = FirefoxBinary(firefox_path=Registry().get('config')['selenium']['firefox_path'], log_file=fo) try: self.browser = SeleniumBrowser( profile, firefox_binary=binary, ddos_phrase=self.ddos_phrase, proxy=proxy, ddos_human=self.ddos_human, ) except WebDriverException as e: self.logger.ex(e) self.logger.log("Re-trying. Browser creation error: " + str(e)) shutil.rmtree(profile_path) time.sleep(5) retry_counter += 1 return self.browser_create(retry_counter) self.browser.set_page_load_timeout(Registry().get('config')['selenium']['timeout_page_load']) self.browser.implicitly_wait(Registry().get('config')['selenium']['timeout_page_load'])