def __fetch_requests(self, task_entry, shell_groups):
p = Pattern(task_entry, self.__getCurrentShell(task_entry, shell_groups), self.global_data)
timeout = task_entry.get('timeout', 120)
urls = p.convertPattern('url')
s = requests.session()
headers = task_entry.get('headers', [])
task_entry['datas'] = []
if not urls:
return task_entry
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
try:
response = s.get(url, timeout=timeout, headers=headers)
if 200 != response.status_code:
self.logger.error("fetch " + url + " failed with code " + (str)(response.status_code))
data = response.text
except:
self.logger.error("fetch " + url + " failed in sockets")
task_entry['datas'].append(data)
return task_entry
def __fetch_requests(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data)
timeout = task_entry.get('timeout', 120)
urls = p.convertPattern('url')
s = requests.session()
headers = task_entry.get('headers', [])
task_entry['datas'] = []
if not urls:
return task_entry
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
try:
response = s.get(url, timeout=timeout, headers=headers)
if 200 != response.status_code:
self.logger.error("fetch " + url + " failed with code " + (str)(response.status_code))
data = response.text
except:
self.logger.error("fetch " + url + " failed in sockets")
task_entry['datas'].append(data)
return task_entry
def __fetch(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.config_data)
urls = p.convertPattern('url')
s = requests.session()
task_entry['datas'] = []
for url in urls:
self.logger.info("fetching " + url)
if "" == url:
# do not fetch null url
continue
response = s.get(url)
if 200 != response.status_code:
self.logger.error("fetch " + url + " failed with code" + response.status_code)
data = response.text
task_entry['datas'].append(data)
return task_entry
def __fetch_webkit(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry),
self.global_data)
import cwebbrowser
task_entry['datas'] = []
urls = p.convertPattern('url')
timeout = task_entry.get('timeout', 120)
delay = task_entry.get('delay', 0)
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
browser = cwebbrowser.CWebBrowser()
browser.setHeaders(task_entry.get('headers', []))
#browser.show();
try:
browser.load(url=url, load_timeout=timeout, delay=delay)
except cwebbrowser.Timeout:
self.logger.error("fetch " + url + " timeout ")
except Exception, exception:
self.logger.error("fetch " + url + " error ")
print "Exception message:", exception
else:
html = browser.html()
if html:
html = html.encode('utf-8')
data = html
else:
self.logger.error("fetch " + url +
" failed with no response")
task_entry['datas'].append(data)
browser.close()
def __fetch_webkit(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data)
import cwebbrowser
task_entry['datas'] = []
urls = p.convertPattern('url')
timeout = task_entry.get('timeout', 120)
delay = task_entry.get('delay', 0)
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
browser = cwebbrowser.CWebBrowser()
browser.setHeaders(task_entry.get('headers', []))
#browser.show();
try:
browser.load(url=url, load_timeout=timeout, delay=delay)
except cwebbrowser.Timeout:
self.logger.error("fetch " + url + " timeout ")
except Exception, exception:
self.logger.error("fetch " + url + " error ")
print "Exception message:", exception
else:
html = browser.html()
if html:
html = html.encode('utf-8')
data = html
else:
self.logger.error("fetch " + url + " failed with no response")
task_entry['datas'].append(data)
browser.close()
