def run(self):
blocked_urls = []
if self.browser == 'chrome':
browser = chrome()
else:
browser = phantomjs()
# add cookie, the scope is url_list[0]'s top-domain
add_cookie(browser, self.url_list[0])
for url in self.url_list:
splited = url.split('/', 3)
path = '/'.join(splited)
# if not block
if path not in blocked_urls:
try:
browser.get(url)
except TimeoutException, e:
print e
# save if browser get() exception
REQUEST_ERROR.append(('Render get()', url, 'timeout'))
# browser blocks sometimes.
rtn = self.handle_block(browser)
if rtn is not None:
browser = rtn
splited = url.split('/', 3)
path = '/'.join(splited)
blocked_urls.append(path)
except BadStatusLine, e:
print e
REQUEST_ERROR.append(
('Render get()', url, 'BadStatusLine'))
splited = url.split('/', 3)
path = '/'.join(splited)
blocked_urls.append(path)
except UnicodeDecodeError:
pass
def make_request(method, url, headers, body):
domain = get_domain_from_url(url)
if headers:
# delete some needless header
for key in headers.keys():
if key in [
'Accept-Encoding', 'Content-Type', 'Accept-Language',
'Accept', 'Connection'
]:
del headers[key]
else:
headers = getheader_dict(domain)
# proxy(127.0.0.1:8080)
# opener=urllib2.build_opener(proxy_support)
# opener = urllib2.build_opener()
# opener.addheaders=headers
# urllib2.install_opener(opener)
if method == 'GET':
req = urllib2.Request(url, headers=headers)
try:
resp = urllib2.urlopen(req)
# save redirect
if resp.url != url:
REDIRECT.append(url)
return resp
except URLError, e:
REQUEST_ERROR.append(('make_request()', url, e.reason))
except CertificateError:
REQUEST_ERROR.append(
('make_request()', url, 'ssl.CertificateError'))
def gen_traffic(self, url):
domain = get_domain_from_url(url)
# add cookie to DEFAULT_HEADER
cookie = get_cookie(domain)
self.DEFAULT_HEADER['Cookie'] = cookie
# add referer
self.DEFAULT_HEADER['Referer'] = 'https"//' + domain + '/'
request = HttpRequest(method='GET', url=url, headers=self.DEFAULT_HEADER, body='')
req = urllib2.Request(url=url, headers=self.DEFAULT_HEADER)
with gevent.Timeout(10, False)as t:
try:
resp = urllib2.urlopen(req)
except urllib2.URLError, e:
REQUEST_ERROR.append(('gen_traffic()', url, e.reason))
except CertificateError:
REQUEST_ERROR.append(('gen_traffic()', url, 'ssl.CertificateError'))
def run(self):
blocked_urls = []
if self.browser == 'chrome':
browser = chrome()
elif self.browser == 'chrome-headless':
browser = chrome(headless=True)
else:
browser = phantomjs()
# add cookie, the scope is case_list[0].url's top-domain
add_cookie(browser, case_list[0].url)
for case in self.case_list:
if case.method == 'POST':
continue
vul = case.vul
url = case.url
args = case.args
splited = url.split('/', 3)
path = '/'.join(splited)
# if not block
if path not in blocked_urls:
try:
browser.get(url)
except TimeoutException, e:
LOGGER.warn(e)
# mark if browser get() exception
REQUEST_ERROR.append(('Openner get()', url, 'timeout'))
# browser blocked sometimes.
rtn = self.handle_block(browser)
if rtn is not None:
browser = rtn
splited = url.split('/', 3)
path = '/'.join(splited)
blocked_urls.append(path)
except BadStatusLine, e:
LOGGER.warn(e)
REQUEST_ERROR.append(
('Render get()', url, 'BadStatusLine'))
splited = url.split('/', 3)
path = '/'.join(splited)
blocked_urls.append(path)
('make_request()', url, 'ssl.CertificateError'))
except ValueError, e:
print e
except BadStatusLine, e:
print e
except SocketError, e:
print e
elif method == 'POST':
req = urllib2.Request(url, data=body, headers=headers)
try:
resp = urllib2.urlopen(req)
if resp.url != url:
REDIRECT.append(url)
return resp
except URLError, e:
REQUEST_ERROR.append(('make_request()', url, e.reason))
except CertificateError:
REQUEST_ERROR.append(
('make_request()', url, 'ssl.CertificateError'))
except ValueError, e:
print e
except BadStatusLine, e:
print e
except SocketError, e:
print e
def chrome(headless=False):
# support to get response status and headers
d = DesiredCapabilities.CHROME
d['loggingPrefs'] = {'performance': 'ALL'}