Exemple #1
0
 def run(self):
     blocked_urls = []
     if self.browser == 'chrome':
         browser = chrome()
     else:
         browser = phantomjs()
     # add cookie, the scope is url_list[0]'s top-domain
     add_cookie(browser, self.url_list[0])
     for url in self.url_list:
         splited = url.split('/', 3)
         path = '/'.join(splited)
         # if not block
         if path not in blocked_urls:
             try:
                 browser.get(url)
             except TimeoutException, e:
                 print e
                 # save if browser get() exception
                 REQUEST_ERROR.append(('Render get()', url, 'timeout'))
                 # browser blocks sometimes.
                 rtn = self.handle_block(browser)
                 if rtn is not None:
                     browser = rtn
                     splited = url.split('/', 3)
                     path = '/'.join(splited)
                     blocked_urls.append(path)
             except BadStatusLine, e:
                 print e
                 REQUEST_ERROR.append(
                     ('Render get()', url, 'BadStatusLine'))
                 splited = url.split('/', 3)
                 path = '/'.join(splited)
                 blocked_urls.append(path)
             except UnicodeDecodeError:
                 pass
Exemple #2
0
def make_request(method, url, headers, body):
    domain = get_domain_from_url(url)
    if headers:
        # delete some needless header
        for key in headers.keys():
            if key in [
                    'Accept-Encoding', 'Content-Type', 'Accept-Language',
                    'Accept', 'Connection'
            ]:
                del headers[key]
    else:
        headers = getheader_dict(domain)
    # proxy(127.0.0.1:8080)
    # opener=urllib2.build_opener(proxy_support)
    # opener = urllib2.build_opener()
    # opener.addheaders=headers
    # urllib2.install_opener(opener)
    if method == 'GET':
        req = urllib2.Request(url, headers=headers)
        try:
            resp = urllib2.urlopen(req)
            # save redirect
            if resp.url != url:
                REDIRECT.append(url)
            return resp
        except URLError, e:
            REQUEST_ERROR.append(('make_request()', url, e.reason))
        except CertificateError:
            REQUEST_ERROR.append(
                ('make_request()', url, 'ssl.CertificateError'))
Exemple #3
0
 def gen_traffic(self, url):
     domain = get_domain_from_url(url)
     # add cookie to DEFAULT_HEADER
     cookie = get_cookie(domain)
     self.DEFAULT_HEADER['Cookie'] = cookie
     # add referer
     self.DEFAULT_HEADER['Referer'] = 'https"//' + domain + '/'
     request = HttpRequest(method='GET', url=url, headers=self.DEFAULT_HEADER, body='')
     req = urllib2.Request(url=url, headers=self.DEFAULT_HEADER)
     with gevent.Timeout(10, False)as t:
         try:
             resp = urllib2.urlopen(req)
         except urllib2.URLError, e:
             REQUEST_ERROR.append(('gen_traffic()', url, e.reason))
         except CertificateError:
             REQUEST_ERROR.append(('gen_traffic()', url, 'ssl.CertificateError'))
Exemple #4
0
 def run(self):
     blocked_urls = []
     if self.browser == 'chrome':
         browser = chrome()
     elif self.browser == 'chrome-headless':
         browser = chrome(headless=True)
     else:
         browser = phantomjs()
     # add cookie, the scope is case_list[0].url's top-domain
     add_cookie(browser, case_list[0].url)
     for case in self.case_list:
         if case.method == 'POST':
             continue
         vul = case.vul
         url = case.url
         args = case.args
         splited = url.split('/', 3)
         path = '/'.join(splited)
         # if not block
         if path not in blocked_urls:
             try:
                 browser.get(url)
             except TimeoutException, e:
                 LOGGER.warn(e)
                 # mark if browser get() exception
                 REQUEST_ERROR.append(('Openner get()', url, 'timeout'))
                 # browser blocked sometimes.
                 rtn = self.handle_block(browser)
                 if rtn is not None:
                     browser = rtn
                     splited = url.split('/', 3)
                     path = '/'.join(splited)
                     blocked_urls.append(path)
             except BadStatusLine, e:
                 LOGGER.warn(e)
                 REQUEST_ERROR.append(
                     ('Render get()', url, 'BadStatusLine'))
                 splited = url.split('/', 3)
                 path = '/'.join(splited)
                 blocked_urls.append(path)
Exemple #5
0
                ('make_request()', url, 'ssl.CertificateError'))
        except ValueError, e:
            print e
        except BadStatusLine, e:
            print e
        except SocketError, e:
            print e
    elif method == 'POST':
        req = urllib2.Request(url, data=body, headers=headers)
        try:
            resp = urllib2.urlopen(req)
            if resp.url != url:
                REDIRECT.append(url)
            return resp
        except URLError, e:
            REQUEST_ERROR.append(('make_request()', url, e.reason))
        except CertificateError:
            REQUEST_ERROR.append(
                ('make_request()', url, 'ssl.CertificateError'))
        except ValueError, e:
            print e
        except BadStatusLine, e:
            print e
        except SocketError, e:
            print e


def chrome(headless=False):
    # support to get response status and headers
    d = DesiredCapabilities.CHROME
    d['loggingPrefs'] = {'performance': 'ALL'}