def __init__(self):
self.tags = {}
tags = Registry().get('config')['spider']['tags']
tags = tags.split(",")
for tag in tags:
tag = tag.split("|")
self.tags[tag[0]] = tag[1]
def __init__(self):
self.tags = {}
tags = Registry().get('config')['spider']['tags']
tags = tags.split(",")
for tag in tags:
tag = tag.split("|")
self.tags[tag[0]] = tag[1]
def browser_create(self, retry_counter=0):
""" Create a browser """
if retry_counter >= int(Registry().get('config')['selenium']
['browser_recreate_errors_limit']):
raise Exception(
"WebDriver can`t create browser. Check errors log selenium settings."
)
self_num = random.randint(0, 99999)
myProxy = Registry().get('proxies').get_proxy()
if myProxy:
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': ''
})
self.proxy_using = True
else:
#print "No proxy"
proxy = None
self.proxy_using = False
profile_path = '/tmp/wr-selenium-{0}/'.format(self_num)
if os.path.exists(profile_path):
shutil.rmtree(profile_path)
if not os.path.exists(profile_path):
os.mkdir(profile_path)
profile = webdriver.FirefoxProfile(profile_path)
if Registry().get('config')['selenium']['css_load'] != '1':
profile.set_preference('permissions.default.stylesheet', 2)
if Registry().get('config')['selenium']['images_load'] != '1':
profile.set_preference('permissions.default.image', 2)
if Registry().get('config')['selenium']['flash_load'] != '1':
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
profile.set_preference("browser.startup.homepage", "about:blank")
profile.set_preference("browser.formfill.enable", "false")
profile.set_preference("startup.homepage_welcome_url", "about:blank")
profile.set_preference("startup.homepage_welcome_url.additional",
"about:blank")
if myProxy and len(myProxy):
agent_IP, agent_Port = myProxy.split(":")
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.share_proxy_settings", True)
profile.set_preference("network.http.use-cache", False)
profile.set_preference("network.proxy.http", agent_IP)
profile.set_preference("network.proxy.http_port", int(agent_Port))
profile.set_preference('network.proxy.ssl_port', int(agent_Port))
profile.set_preference('network.proxy.ssl', agent_IP)
profile.set_preference('network.proxy.socks', agent_IP)
profile.set_preference('network.proxy.socks_port', int(agent_Port))
fo = open('/tmp/firefox-run-{0}.log'.format(self_num), "w")
binary = FirefoxBinary(
firefox_path=Registry().get('config')['selenium']['firefox_path'],
log_file=fo)
try:
self.browser = SeleniumBrowser(
profile,
firefox_binary=binary,
proxy=proxy,
browser_wait_re=self.browser_wait_re,
)
except WebDriverException as e:
self.logger.ex(e)
self.logger.log("Re-trying. Browser creation error: " + str(e))
shutil.rmtree(profile_path)
time.sleep(5)
retry_counter += 1
return self.browser_create(retry_counter)
self.browser.set_page_load_timeout(
Registry().get('config')['selenium']['timeout_page_load'])
self.browser.implicitly_wait(
Registry().get('config')['selenium']['timeout_page_load'])
def browser_create(self, retry_counter = 0):
""" Create a browser """
if retry_counter >= int(Registry().get('config')['selenium']['browser_recreate_errors_limit']):
raise Exception("WebDriver can`t create browser. Check errors log selenium settings.")
self_num = random.randint(0, 99999)
myProxy = Registry().get('proxies').get_proxy()
if myProxy:
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': ''
})
self.proxy_using = True
else:
#print "No proxy"
proxy = None
self.proxy_using = False
profile_path = '/tmp/wr-selenium-{0}/'.format(self_num)
if os.path.exists(profile_path):
shutil.rmtree(profile_path)
if not os.path.exists(profile_path):
os.mkdir(profile_path)
profile = webdriver.FirefoxProfile(profile_path)
if Registry().get('config')['selenium']['css_load'] != '1':
profile.set_preference('permissions.default.stylesheet', 2)
if Registry().get('config')['selenium']['images_load'] != '1':
profile.set_preference('permissions.default.image', 2)
if Registry().get('config')['selenium']['flash_load'] != '1':
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
profile.set_preference("browser.startup.homepage", "about:blank")
profile.set_preference("startup.homepage_welcome_url", "about:blank")
profile.set_preference("startup.homepage_welcome_url.additional", "about:blank")
if myProxy and len(myProxy):
agent_IP, agent_Port = myProxy.split(":")
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.share_proxy_settings", True)
profile.set_preference("network.http.use-cache", False)
profile.set_preference("network.proxy.http", agent_IP)
profile.set_preference("network.proxy.http_port", int(agent_Port))
profile.set_preference('network.proxy.ssl_port', int(agent_Port))
profile.set_preference('network.proxy.ssl', agent_IP)
profile.set_preference('network.proxy.socks', agent_IP)
profile.set_preference('network.proxy.socks_port', int(agent_Port))
fo = open('/tmp/firefox-run-{0}.log'.format(self_num), "w")
binary = FirefoxBinary(firefox_path=Registry().get('config')['selenium']['firefox_path'], log_file=fo)
try:
self.browser = SeleniumBrowser(
profile,
firefox_binary=binary,
ddos_phrase=self.ddos_phrase,
proxy=proxy,
ddos_human=self.ddos_human,
)
except WebDriverException as e:
self.logger.ex(e)
self.logger.log("Re-trying. Browser creation error: " + str(e))
shutil.rmtree(profile_path)
time.sleep(5)
retry_counter += 1
return self.browser_create(retry_counter)
self.browser.set_page_load_timeout(Registry().get('config')['selenium']['timeout_page_load'])
self.browser.implicitly_wait(Registry().get('config')['selenium']['timeout_page_load'])
