def __fetch(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data)
timeout = task_entry.get('timeout', 120)
urls = p.convertPattern('url')
s = requests.session()
if task_entry.get('cookie'):
cookie_str = p.convertPattern('cookie')
cookie_str_arr = cookie_str[0].split("&")
for str_param in cookie_str_arr:
cookie_params = str_param.split("=")
s.cookies.set(cookie_params[0], cookie_params[1])
task_entry['datas'] = []
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
try:
response = s.get(url, timeout=timeout)
if 200 != response.status_code:
self.logger.error("fetch " + url + " failed with code " + (str)(response.status_code))
data = response.text
except:
self.logger.error("fetch " + url + " failed in sockets")
task_entry['datas'].append(data)
return task_entry
def __fetch_requests(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data)
timeout = task_entry.get('timeout', 120)
urls = p.convertPattern('url')
s = requests.session()
headers = task_entry.get('headers', [])
task_entry['datas'] = []
if not urls:
return task_entry
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
try:
response = s.get(url, timeout=timeout, headers=headers)
if 200 != response.status_code:
self.logger.error("fetch " + url + " failed with code " + (str)(response.status_code))
data = response.text
except:
self.logger.error("fetch " + url + " failed in sockets")
task_entry['datas'].append(data)
return task_entry
def __fetch(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.config_data)
urls = p.convertPattern('url')
s = requests.session()
task_entry['datas'] = []
for url in urls:
self.logger.info("fetching " + url)
if "" == url:
# do not fetch null url
continue
response = s.get(url)
if 200 != response.status_code :
self.logger.error("fetch " + url +" failed with code " + (str)(response.status_code))
data = response.text
task_entry['datas'].append(data)
return task_entry
def __fetch_webkit(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data)
import cwebbrowser
task_entry['datas'] = []
urls = p.convertPattern('url')
timeout = task_entry.get('timeout', 120)
delay = task_entry.get('delay', 0)
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
browser = cwebbrowser.CWebBrowser()
browser.setHeaders(task_entry.get('headers', []))
#browser.show();
try:
browser.load(url=url, load_timeout=timeout, delay=delay)
except cwebbrowser.Timeout:
self.logger.error("fetch " + url + " timeout ")
except Exception, exception:
self.logger.error("fetch " + url + " error ")
print "Exception message:", exception
else:
html = browser.html()
if html:
html = html.encode('utf-8')
data = html
else:
self.logger.error("fetch " + url + " failed with no response")
task_entry['datas'].append(data)
browser.close()
def __fetch_webkit(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data)
import cwebbrowser
task_entry['datas'] = []
urls = p.convertPattern('url')
timeout = task_entry.get('timeout', 120)
delay = task_entry.get('delay', 0)
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
browser = cwebbrowser.CWebBrowser()
browser.setHeaders(task_entry.get('headers', []))
#browser.show();
try:
browser.load(url=url, load_timeout=timeout, delay=delay)
except cwebbrowser.Timeout:
self.logger.error("fetch " + url + " timeout ")
except Exception, exception:
self.logger.error("fetch " + url + " error ")
print "Exception message:", exception
else:
html = browser.html()
if html:
html = html.encode('utf-8')
data = html
else:
self.logger.error("fetch " + url + " failed with no response")
task_entry['datas'].append(data)
browser.close()
def __fetch_requests(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data)
timeout = task_entry.get('timeout', 120)
urls = p.convertPattern('url')
s = requests.session()
headers = task_entry.get('headers', [])
task_entry['datas'] = []
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
try:
response = s.get(url, timeout=timeout, headers=headers)
if 200 != response.status_code:
self.logger.error("fetch " + url + " failed with code " + (str)(response.status_code))
data = response.text
except:
self.logger.error("fetch " + url + " failed in sockets")
task_entry['datas'].append(data)
return task_entry
