Пример #1
0
        def is_present(mpkt):
            hdrs = mpkt.cfields.get('dissector.http.headers', None)

            if not hdrs:
                return False

            if 'cookie' in hdrs:
                for cstr in hdrs['cookie']:
                    cookie = SimpleCookie()
                    cookie.load(cstr)

                    for k, mar in cookie.items():
                        cookies[k].append(mar.value)

                return True

            elif 'set-cookie' in hdrs:
                for cstr in hdrs['set-cookie']:
                    cookie = SimpleCookie()
                    cookie.load(cstr)

                    for k, mar in cookie.items():
                        cookies[k].append(mar.value)

                return True

            return False
Пример #2
0
        def is_present(mpkt):
            hdrs = mpkt.cfields.get('dissector.http.headers', None)

            if not hdrs:
                return False

            if 'cookie' in hdrs:
                for cstr in hdrs['cookie']:
                    cookie = SimpleCookie()
                    cookie.load(cstr)

                    for k, mar in cookie.items():
                        cookies[k].append(mar.value)

                return True

            elif 'set-cookie' in hdrs:
                for cstr in hdrs['set-cookie']:
                    cookie = SimpleCookie()
                    cookie.load(cstr)

                    for k, mar in cookie.items():
                        cookies[k].append(mar.value)

                return True

            return False
Пример #3
0
 def parseStrToCookies(self, cookieStr):
     SC = SimpleCookie()
     SC.load(cookieStr)
     cookies = {}
     for key, morsel in SC.items():
         cookies[key] = morsel.value
     return cookies
Пример #4
0
 def cookies(self):
     """Returns a dictionary mapping cookie names to their values."""
     if self._cookies is None:
         c = SimpleCookie(self.environ.get('HTTP_COOKIE'))
         self._cookies = dict([(k.decode('utf-8'), v.value.decode('utf-8'))
                               for k, v in c.items()])
     return self._cookies
Пример #5
0
def get_cookies(str):
    cookie = SimpleCookie()
    cookie.load(str)
 
    cookies = {}
    for key, morsel in cookie.items():
        cookies[key] = morsel.value
    return cookies
Пример #6
0
def get_cookies(str):
    cookie = SimpleCookie()
    cookie.load(str)

    cookies = {}
    for key, morsel in cookie.items():
        cookies[key] = morsel.value
    return cookies
Пример #7
0
    def cookies(self, rawdata):
        """Set Response Cookies.

        :param rawdata: (str or dict). List of `cookielib.Cookie`.
        """
        sc = SimpleCookie()
        sc.load(rawdata)
        self._cookies = sc.items()
Пример #8
0
    def cookies(self, rawdata):
        """Set Response Cookies.

        :param rawdata: (str or dict). List of `cookielib.Cookie`.
        """
        sc = SimpleCookie()
        sc.load(rawdata)
        self._cookies = sc.items()
Пример #9
0
def get_cookies_ignore_origin(response):
    cookies = dict_from_cookiejar(response.cookies)

    if 0 == len(cookies) and SET_COOKIE_KEY in response.headers:
        simple_cookie = SimpleCookie(response.headers[SET_COOKIE_KEY])
        for key, morsel in simple_cookie.items():
            cookies[key] = morsel.value

    return cookies
Пример #10
0
    def __retrieve_cookie_values(self, raw_string):
        cookies = {}
        cookie = SimpleCookie()
        cookie.load(raw_string)

        for key, morsel in cookie.items():
            if any(name.lower() in key.lower()
                   for name in Scanner.cookie_keys):
                cookies[key] = morsel.value

        return cookies
Пример #11
0
def read_sparo_html(url):
    import requests

    coockie = {}

    cookies = {
        'sucuri_cloudproxy_uuid_53a9650e8':
        '6c3d2f947083eaa2a62370d6c90b4495',
        'laravel_session':
        'xC5WAE2yuuU0L8k8pCYUQZGh6YMbsv0vjYNvhGBN',
        'XSRF-TOKEN':
        'eyJpdiI6IjB3OHhLaUNzTEpSXC83ZUtZYnBQVUtnPT0iLCJ2YWx1ZSI6ImtNQWM4b2ZLSE4yN3l0VHFCNEIyMVNmQ2s5THdNdW8zS21tQmluVFVZNDFSNlpZM3UwRkptd1wvMWxnMHBVT09PM3BlMTU1TVVRdm02S3pCUXZlVjU4Zz09IiwibWFjIjoiYzcxZTM4MDU4ZmMxMGE5NjljMTdiMzBjZDI0ODcyOTEzNDc1NDBjNmU5MGQwN2JiZTZiMmNjNjZkOGUwMjhlMSJ9',
    }

    headers = {
        'Host': Domain,
        'User-Agent':
        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',
        'Accept':
        'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'he,he-IL;q=0.8,en-US;q=0.5,en;q=0.3',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1',
        'Pragma': 'no-cache',
        'Cache-Control': 'no-cache',
    }

    x = requests.get(url, headers=headers).content

    if 'var s={}' in x:

        coockie2, token = (get_c(x))
        cookie = SimpleCookie()
        cookie.load(str(coockie2))

        # Even though SimpleCookie is dictionary-like, it internally uses a Morsel object
        # which is incompatible with requests. Manually construct a dictionary instead.
        cookies = {}
        for key, morsel in cookie.items():
            cookies[key] = morsel.value
        cookies['XSRF-TOKEN'] = str(token)
        cookies['max-age'] = '86400'

        #cookies['X-CSRF-TOKEN'] = "wYZBYILaf7Dbw0TZDKjJ3udNO8aPUa7Bns3EQcUe"
        #cookies['XSRF-TOKEN'] = "eyJpdiI6IkVZSmViYXpCdWZwMThjSmJ6dVwvRUhRPT0iLCJ2YWx1ZSI6IkpPQkE5QW83QWc1NzMxY2VwbGR1NkM3RlRDOTJ3dGpXaFF6Q29GWENlY0NOQ3NzSTBTK2ZRN2xPeDJ6dnh5TVlnNEQwbGgweHkwXC85djllSkZ0WTVOQT09IiwibWFjIjoiNTg5Y2U1OTIzY2E0YTM5ZTYxMTY2NjBhOTI4NjhjN2MyMGU1ZDJmM2U0MmEzNDBkYThmNDQ4YzU1M2NkODY5ZCJ9"
        #cookies['laravel_session'] = "xC5WAE2yuuU0L8k8pCYUQZGh6YMbsv0vjYNvhGBN"

        x = requests.get(url, headers=headers, cookies=cookies)

        x.encoding = 'utf-8'
        x = x.content

    return x
Пример #12
0
def test_get_json(yahoo_response, cookies):
    response = yahoo_response('v2/groups/groupname/files/a/2?param1=c&param2=4', {'result': 'returned data'})

    yga = YahooGroupsAPI('groupname', cookies)
    json = yga.get_json('files', 'a', 2, param1='c', param2=4)

    request = response.calls[0].request
    request_cookies = SimpleCookie()
    request_cookies.load(request.headers['Cookie'])
    assert dict(cookies) == {k: v.value for k, v in request_cookies.items()}

    assert json == {'result': 'returned data'}
Пример #13
0
 def use_credentials(self, credentials):
     fetch_arguments = build_fetch_arguments("/")
     credentials(fetch_arguments)
     # Selenium doesn't support arbitrary headers (afaik), so we only
     # support cookies for now.
     if "Cookie" in fetch_arguments.headers:
         cookie = SimpleCookie()
         cookie.load(fetch_arguments.headers["Cookie"])
         for morsel_name, morsel in cookie.items():
             self._driver.add_cookie({
                 "name": morsel_name,
                 "value": morsel.value
             })
Пример #14
0
 def get_dict_cookies(self, rawdata):
     '''Returnt dict from cookies raw text'''
     try:
         from Cookie import SimpleCookie
         cookie = SimpleCookie()
         cookie.load(rawdata)
         cookies = {}
         for key, morsel in cookie.items():
             cookies[key] = morsel.value
     except:
         logging.error('Cannot get cookies from raw text', exc_info=True)
         cookies = {}
     return cookies
Пример #15
0
 def _make_request(self, step, **kwargs):
     request_metadata = self._get_step_metadata(step)
     log.debug('kwargs are : ' + str(kwargs))
     data = None
     if not request_metadata:
         return None
     if 'data' in request_metadata:
         data = request_metadata['data']
     if 'method' in request_metadata:
         method = request_metadata['method']
     else:
         method = 'GET' if not data else 'POST'
     if 'redirect' in request_metadata:
         redirect = request_metadata['redirect']
     else:
         redirect = True
     headers = {}
     if 'csrf_token' in self.cookies and self.cookies['csrf_token']:
         headers['X-CSRF-Token'] = self.cookies['csrf_token']
         log.debug('setting csrf header to ' + self.cookies['csrf_token'])
     if self.cookies:
         cookie_str = "; ".join(
             [str(x) + "=" + str(y) for x, y in self.cookies.items()])
         headers['Cookie'] = cookie_str
     request_params = {
         'url': self._get_step_url(step, **kwargs),
         'method': method,
         'allow_redirects': redirect,
         'headers': headers,
         'verify': False
     }
     if (data):
         if isinstance(data, dict):
             data = {
                 k: v(self, **kwargs) if callable(v) else v
                 for k, v in data.items()
             }
             log.debug('posting data : ' + str(data))
             request_params['data'] = json.dumps(data)
             headers['Content-Type'] = 'application/json'
         else:
             request_params['data'] = data
     r = web.request(**request_params)
     if (r.status_code >= 200 and r.status_code <= 400
             and 'set-cookie' in r.headers):
         cookies = SimpleCookie()
         cookies.load(r.headers['Set-Cookie'])
         for key, value in cookies.items():
             self.cookies[key] = value.value
     return r
Пример #16
0
def middleware(environ, start_response):
    request = Request(environ).copy()

    response = request.get_response(django)

    if response.status_int != 305:
        return response(environ, start_response)
    data = json.loads(response.body)

    proxy = RemoteProxy([data['base_url']], rewrite_links=True)

    request.environ.pop("HTTP_ACCEPT_ENCODING", None)

    request.environ['SCRIPT_NAME'] = str(data['script_name']).rstrip("/")
    request.environ['PATH_INFO'] = "/" + str(data['path_info'].lstrip("/") )

    from Cookie import SimpleCookie as Cookie
    orig_cookie = cookie = request.environ.get('HTTP_COOKIE')
    if cookie:
        cookie = Cookie(cookie)
        for key in data['cookie_blacklist']:
            cookie.pop(key, None)
        cookiestr = []
        for key, value in cookie.items():
            cookiestr.append(value.OutputString())
        cookiestr = "; ".join(cookiestr)
        request.environ['HTTP_COOKIE'] = cookiestr

    request.environ['HTTP_X_THING_THEME'] = data['theme']

    filter = deliverance(proxy)

    def per_project_theme(environ):
        return "%(wsgi.url_scheme)s://%(HTTP_HOST)s%(HTTP_X_THING_THEME)s" % environ
    filter.default_theme = per_project_theme
    filter.rule_getter = TemplateRuleGetter(data['deliverance_rules'])
    filter.use_internal_subrequest = lambda *args, **kw: False

    base_subrequest = filter.build_external_subrequest
    def build_external_subrequest(url, orig_req, log):
        subreq = base_subrequest(url, orig_req, log)
        if url.endswith("/theme/") and orig_cookie:
            subreq.environ['HTTP_COOKIE'] = orig_cookie
        return subreq
    filter.build_external_subrequest = build_external_subrequest

    resp = request.get_response(filter)

    return resp(request.environ, start_response)
Пример #17
0
    def process_cookie(self, cookie, host):
        parsed = SimpleCookie(str(cookie))

        for key, c in parsed.items():
            if b"domain" in c:
                host = c['domain']

            url = self.create_url(host)
            host_tld = self.tld_parse(url)

            self.send_cookie_to_listeners(
                CookieData(name=c.key,
                           value=c.value,
                           domain="." + host_tld,
                           url=url,
                           expiry=self.default_expiry))
Пример #18
0
    def decompose_incoming_envelope(self, ctx, message):
        assert message == FlatDictDocument.REQUEST

        ctx.transport.itself.decompose_incoming_envelope(self, ctx, message)

        if self.parse_cookie:
            cookies = ctx.in_header_doc.get('cookie', [])
            for cookie_string in cookies:
                cookie = SimpleCookie()
                cookie.load(cookie_string)
                for k,v in cookie.items():
                    l = ctx.in_header_doc.get(k, [])
                    l.append(v.coded_value)
                    ctx.in_header_doc[k] = l

        logger.debug('\theader : %r' % (ctx.in_header_doc))
        logger.debug('\tbody   : %r' % (ctx.in_body_doc))
Пример #19
0
    def decompose_incoming_envelope(self, ctx, message):
        assert message == FlatDictDocument.REQUEST

        ctx.transport.itself.decompose_incoming_envelope(self, ctx, message)

        if self.parse_cookie:
            cookies = ctx.in_header_doc.get('cookie', [])
            for cookie_string in cookies:
                cookie = SimpleCookie()
                cookie.load(cookie_string)
                for k,v in cookie.items():
                    l = ctx.in_header_doc.get(k, [])
                    l.append(v.coded_value)
                    ctx.in_header_doc[k] = l

        logger.debug('\theader : %r' % (ctx.in_header_doc))
        logger.debug('\tbody   : %r' % (ctx.in_body_doc))
Пример #20
0
    def process_request(self, request, spider):
        """Process a request using the selenium driver if applicable"""
        # print 'SeleniumMiddleware process_request: request ' + str(request)
        if not request.meta.get('enable_selenium', False):
            return None

        cookie = SimpleCookie()
        cookie.load(request.headers.getlist('Cookie')[0])

        driver = self.generate_webdriver()

        for cookie_name, morsel in cookie.items():
            driver.add_cookie({'name': cookie_name, 'value': morsel.value})

        driver.get(request.url)

        # wait til page loads
        # print driver.find_element_by_tag_name(
        #     'title').get_attribute(
        #         'innerHTML') + ', ' + request.meta['title']
        # print 'compare: ' + str(driver.find_element_by_tag_name(
        #     'title').get_attribute(
        #         'innerHTML') == request.meta['title'])
        # print EC.text_to_be_present_in_element(
        #     (By.TAG_NAME, 'title'), request.meta['title'])
        try:
            element = WebDriverWait(
                driver,
                10).until(lambda x: driver.find_element_by_tag_name('title').
                          get_attribute('innerHTML') == request.meta['title'])
        except TimeoutException:
            raise IgnoreRequest('Cannot open: ' + request.url)

        # print 'waiting ends: ' + request.meta['title']
        body = driver.page_source

        # Expose the driver via the "meta" attribute
        request.meta.update({'driver': driver})

        # print 'send response: ' + str(request)

        return HtmlResponse(request.url,
                            body=body,
                            encoding='utf-8',
                            request=request)
Пример #21
0
 def probe_securecookie(self):
     """Something that looks like a securecookie was detected"""
     rv = self.make_request(self.config.url)
     setcookie = rv.headers.get('set-cookie')
     if setcookie is None:
         return False
     if not '; Path=' in setcookie:
         return False
     cookie = SimpleCookie(setcookie)
     for key, morsel in cookie.items():
         if not '?' in morsel.value:
             continue
         try:
             morsel.value.split('?')[0].decode('base64')
             return True
         except Exception:
             return False
     return False
Пример #22
0
 def probe_securecookie(self):
     """Something that looks like a securecookie was detected"""
     rv = self.make_request(self.config.url)
     setcookie = rv.headers.get('set-cookie')
     if setcookie is None:
         return False
     if not '; Path=' in setcookie:
         return False
     cookie = SimpleCookie(setcookie)
     for key, morsel in cookie.items():
         if not '?' in morsel.value:
             continue
         try:
             morsel.value.split('?')[0].decode('base64')
             return True
         except Exception:
             return False
     return False
Пример #23
0
        def build(resp):

            response = Response()

            # Pass settings over.
            response.config = self.config

            if resp:

                # Fallback to None if there's no staus_code, for whatever reason.
                response.status_code = getattr(resp, 'status', None)

                # Make headers case-insensitive.
                response.headers = CaseInsensitiveDict(
                    getattr(resp, 'headers', None))

                # Start off with our local cookies.
                cookies = self.cookies or dict()

                # Add new cookies from the server.
                if 'set-cookie' in response.headers:
                    cookie_header = response.headers['set-cookie']

                    c = SimpleCookie()
                    c.load(cookie_header)

                    for k, v in c.items():
                        cookies.update({k: v.value})

                # Save cookies in Response.
                response.cookies = cookies

            # Save original resopnse for later.
            response.raw = resp

            if is_error:
                response.error = resp

            response.url = self.full_url

            return response
Пример #24
0
        def build(resp):

            response = Response()

            # Pass settings over.
            response.config = self.config

            if resp:

                # Fallback to None if there's no staus_code, for whatever reason.
                response.status_code = getattr(resp, 'status', None)

                # Make headers case-insensitive.
                response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None))

                # Start off with our local cookies.
                cookies = self.cookies or dict()

                # Add new cookies from the server.
                if 'set-cookie' in response.headers:
                    cookie_header = response.headers['set-cookie']

                    c = SimpleCookie()
                    c.load(cookie_header)

                    for k,v in c.items():
                        cookies.update({k: v.value})

                # Save cookies in Response.
                response.cookies = cookies

            # Save original resopnse for later.
            response.raw = resp

            if is_error:
                response.error = resp

            response.url = self.full_url

            return response
Пример #25
0
class AbstractCookiesTransport:
    """An Transport subclass that retains cookies over its lifetime and has the
    ability to store/load cookies in a file.

    Taken from http://stackoverflow.com/a/25876504 and modified to implement
    cookie file save/load functionality.
    """

    def __init__(self, cookiefile=None, **kwargs):
        super().__init__(**kwargs)
        self.cookiefile = cookiefile
        self.cookies = SimpleCookie()
        self.load_cookies()

    def load_cookies(self, cookiefile=None):
        cookiefile = cookiefile or self.cookiefile
        if os.path.isfile(cookiefile):
            with open(cookiefile) as fd:
                self.cookies.load(fd.read())

    def save_cookies(self, cookiefile=None):
        cookiefile = cookiefile or self.cookiefile
        with open(cookiefile, 'w') as fd:
            fd.write(self.cookies.output(header="", sep="\n"))

    def send_headers(self, connection, headers):
        if self.cookies:
            cookies = [
                "{}={}".format(k,v.value) for k,v in self.cookies.items()
            ]
            cookie_str = "; ".join(cookies)
            connection.putheader("Cookie", cookie_str)
        super().send_headers(connection, headers)

    def parse_response(self, response):
        cookies = response.msg.get_all("Set-Cookie") or []
        for cookie in cookies:
            self.cookies.load(cookie)  # Add or replace cookie
        self.save_cookies()
        return super().parse_response(response)
Пример #26
0
def requester(target, cookie, user_agent, proxy, match, filename):
    try:
        url = target + filename

        proxy_dict = {}
        cookie_dict = {}

        if proxy != "":
            proxy_dict = {"http": proxy, "https": proxy}

        # If the user specified a cookie convert it to a requests acceptable dict
        if cookie != "":
            cookie_dict = SimpleCookie()
            cookie_dict.load(cookie)
            cookie = {}

            for key, morse1 in cookie_dict.items():
                cookie[key] = morse1.value

        # If the user want to match a specific string in the response
        if match != "":
            req = requests.get(url,
                               cookies=cookie,
                               headers={"User-Agent": user_agent},
                               proxies=proxy_dict,
                               verify=False)
            requester.q.put([url, req.status_code, req.content])
        else:
            req = requests.head(url,
                                cookies=cookie,
                                headers={"User-Agent": user_agent},
                                proxies=proxy_dict,
                                verify=False)
            requester.q.put([url, req.status_code])
    except KeyboardInterrupt:
        sys.stdout = open(os.devnull, 'w')

#convert cookies
from Cookie import SimpleCookie

Cookie11 = 'Cookie: akavpau_p6=1492510799~id=bad5a5f677b9c92fd22ae7ce69a953d7; search.perf.metric=timerPromiseAll=404ms|timerHeaderAction=53ms|timerSearchAction=404ms|timerFooterAction=26ms|timerPreso=402ms; akavpau_p3=1492510790~id=d7334d155b9d5313f20c6c8b19e356d7; AID=wmlspartner%3D0%3Areflectorid%3D0000000000000000000000%3Alastupd%3D1492510198882; DL=98105%2C47.66059875488281%2C-122.29190063476562%2Cip%2C98105%2CUSA%2CWA; prefper=PREFSTORE~13098~2PREFCITY~1Bellevue~2PREFFULLSTREET~115063%20Main%20St~2PREFSTATE~1WA~2PREFZIP~198105; bstc=ZYcj-sZzb56LzolJAq24WY; NSID=3098.8-5939.8-5678.13-2594.13-2516.13-2317.16-5195.16-3053.18-5272.18-2325.19-5628.21-5073.22-2571.23-2385.25-3794.25-3801.29-2595.29-4137.30-3537.34-3757.34; vtc=ZYcj-sZzb56LzolJAq24WY; exp-ck=vOXNpEqkz0MWIxT; exp=0%2B1492510198%2BZYcj-sZzb56LzolJAq24WY%2B0%2BCcvJB.vOXNp|K8foX.Eqkz0|VxHIY.MWIxT; com.wm.reflector="reflectorid:0000000000000000000000@lastupd:1492510198883@firstcreate:1492510198883"'	

output = {'akavpau_p6':'1492510799~id=bad5a5f677b9c92fd22ae7ce69a953d7', 'search.perf.metric':'timerPromiseAll=404ms|timerHeaderAction=53ms|timerSearchAction=404ms|timerFooterAction=26ms|timerPreso=402ms', 'akavpau_p3':'1492510790~id=d7334d155b9d5313f20c6c8b19e356d7', 'AID':'wmlspartner%3D0%3Areflectorid%3D0000000000000000000000%3Alastupd%3D1492510198882', 'DL':'98105%2C47.66059875488281%2C-122.29190063476562%2Cip%2C98105%2CUSA%2CWA', 'prefper':'PREFSTORE~13098~2PREFCITY~1Bellevue~2PREFFULLSTREET~115063%20Main%20St~2PREFSTATE~1WA~2PREFZIP~198105', 'bstc':'ZYcj-sZzb56LzolJAq24WY', 'NSID':'3098.8-5939.8-5678.13-2594.13-2516.13-2317.16-5195.16-3053.18-5272.18-2325.19-5628.21-5073.22-2571.23-2385.25-3794.25-3801.29-2595.29-4137.30-3537.34-3757.34', 'vtc':'ZYcj-sZzb56LzolJAq24WY', 'exp-ck':'vOXNpEqkz0MWIxT', 'exp':'0%2B1492510198%2BZYcj-sZzb56LzolJAq24WY%2B0%2BCcvJB.vOXNp|K8foX.Eqkz0|VxHIY.MWIxT', 'com.wm.reflector':'"reflectorid:0000000000000000000000@lastupd:1492510198883@firstcreate:1492510198883"'}

cookie = SimpleCookie()
cookie.load(Cookie11)

cookies12 = {}
for key, morsel in cookie.items():
    cookies12[key] = morsel.value
	
print cookies12
Пример #28
0
def check_truth(injection_string):
    global data
    if config.request_method == "POST":
        url = config.injection_url
        post_params = deepcopy(config.post_params)
        for key in post_params:
            if "INJECTION_STRING" in post_params[key]:
                post_params[key] = post_params[key].replace('INJECTION_STRING', injection_string)
    else:
        url = "%s and (%s)" % (config.injection_url, injection_string)



    args = {}
    if config.cookies:
        from Cookie import SimpleCookie

        cookie = SimpleCookie()
        cookie.load(config.cookies)
        cookies = {}
        for key, morsel in cookie.items():
            cookies[key] = morsel.value


        args["cookies"] = cookies
    if config.headers:
        args["headers"] = config.headers
    if config.proxies:
        args["proxies"] = config.proxies
    if config.request_method == "POST":
        args["data"] = post_params

    # print injection_string, args['data']

    if config.parameter_needs_quote:
        url = "%s' and (%s)-- j" % (config.injection_url, injection_string)

    if injection_string in data:
        return data[injection_string]

    try:
        if config.request_method == "POST":
            response = requests.post(url, allow_redirects=False, **args)
        else:
            response = requests.get(url, allow_redirects=False, **args)

        if config.truth_check == "STATUS_CODE":
            if response.status_code == config.truth_status:
                lock.acquire()
                data[injection_string] = True
                add_to_cache(injection_string, True)
                lock.release()
                return True
        elif config.truth_check == "RESPONSE_HEADER" and response.status_code == 302:
            if response.headers[config.truth_response_header_name] == config.truth_response_header_value:
                lock.acquire()
                data[injection_string] = True
                add_to_cache(injection_string, True)
                lock.release()
                return True
        elif config.truth_check == "CONTENT_STRING":
            content = response.content
            if config.truth_string in content:
                lock.acquire()
                data[injection_string] = True
                add_to_cache(injection_string, True)
                lock.release()
                return True
    except:
        traceback.print_exc()
        raise Exception("Error occurred while requesting the url '''%s'''!" % url)

    lock.acquire()
    data[injection_string] = False
    add_to_cache(injection_string, False)
    lock.release()
    return False
Пример #29
0
    def do_request(self, req, status, expect_errors):
        """
        Executes the given request (``req``), with the expected
        ``status``.  Generally ``.get()`` and ``.post()`` are used
        instead.

        To use this::

            resp = app.do_request(webtest.TestRequest.blank(
                'url', ...args...))

        Note you can pass any keyword arguments to
        ``TestRequest.blank()``, which will be set on the request.
        These can be arguments like ``content_type``, ``accept``, etc.
        """
        __tracebackhide__ = True
        errors = StringIO()
        req.environ['wsgi.errors'] = errors
        if self.cookies:
            cookie_header = ''.join([
                '%s=%s; ' % (name, cookie_quote(value))
                for name, value in self.cookies.items()])
            req.environ['HTTP_COOKIE'] = cookie_header
        req.environ['paste.testing'] = True
        req.environ['paste.testing_variables'] = {}
        app = lint.middleware(self.app)
        old_stdout = sys.stdout
        out = CaptureStdout(old_stdout)
        try:
            sys.stdout = out
            start_time = time.time()
            ## FIXME: should it be an option to not catch exc_info?
            res = req.get_response(app, catch_exc_info=True)
            end_time = time.time()
        finally:
            sys.stdout = old_stdout
        res.app = app
        res.test_app = self
        # We do this to make sure the app_iter is exausted:
        res.body
        res.errors = errors.getvalue()
        total_time = end_time - start_time
        for name, value in req.environ['paste.testing_variables'].items():
            if hasattr(res, name):
                raise ValueError(
                    "paste.testing_variables contains the variable %r, but "
                    "the response object already has an attribute by that "
                    "name" % name)
            setattr(res, name, value)
        if not expect_errors:
            self._check_status(status, res)
            self._check_errors(res)
        res.cookies_set = {}
        for header in res.headers.getall('set-cookie'):
            try:
                c = SimpleCookie(header)
            except CookieError, e:
                raise CookieError(
                    "Could not parse cookie header %r: %s" % (header, e))
            for key, morsel in c.items():
                self.cookies[key] = morsel.value
                res.cookies_set[key] = morsel.value
Пример #30
0
    parser.add_option("-p", "--proxy", dest="proxy")
    parser.add_option("-s", "--sleep", dest="seconds_to_sleep")
    parser.add_option("-d", "--delay", dest="delay")
    parser.add_option("-c", "--cookies", dest="cookies")
    
    options, args = parser.parse_args()

    target_url = options.target_url
    seconds_to_sleep = float(options.seconds_to_sleep)
    delay = float(options.delay)
    if options.cookies:
        raw_cookies = options.cookies
        simple_cookies = SimpleCookie()
        simple_cookies.load(raw_cookies)
        cookies = {}
        for key, morsel in simple_cookies.items():
            cookies[key] = morsel.value
    else:
        cookies = { "1337": "1337" }
    if options.proxy:
        proxy = { 
            "http": "%s" % options.proxy,
            "https": "%s" % options.proxy
            }
    else:
        proxy = { "http": "None" }

    # ASCII character ranges, ordered by probability
    upper_underscore = list(range(65,90))
    upper_underscore.append(95)
    upper_lower_numeric_symbols = list(range(32,126))
class BrowserPythonNetlocData:
    def __init__(self, browser, netloc):

        # Remove standard port number from netloc string
        # Allowed by HTTP W3C and make some web-apps happy
        if ':' in netloc:
            nl = netloc.split(':')
            if nl[1] == '80' or nl[1] == '443':
                netloc = nl[0]

        self.browser = browser
        self.netloc = netloc

        self.cookies = SimpleCookie()
        self.httpool = {}
        self.lock = Lock()

        self.header = {}
        self.header[
            'User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0 PYTHON'
        self.header['Connection'] = 'keep-alive'
        self.header[
            'Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
        self.header['Accept-Language'] = 'en-US,en;q=0.5'
        self.header['Content-Type'] = 'application/x-www-form-urlencoded'
        self.header['Host'] = self.netloc

    def _get_http_pool(self, scheme):
        if scheme not in self.httpool:
            loc = "%s://%s" % (scheme, self.netloc)
            self.browser.log_debug("allocating connection pool to %s" % loc)
            self.httpool[scheme] = HTTPPool("%s" % loc)
        return self.httpool[scheme]

    def __del__(self):
        for pool in self.httpool.values():
            pool.clear()

    def set_cookie(self, url, key, val=None, path=None):
        self.browser.log_debug("set cookie: %s, %s=%s" % (url, key, str(val)))

        scheme, netloc, path = parse_url(url)
        with self.lock:
            if type(key) == 'dict':
                c = key
                self.cookies[c['name']] = c
            elif val:
                self.cookies[key] = val
                if path:
                    self.cookies[key]["path"] = path
            elif path in self.cookies:
                del self.cookies[key]

    def set_header(self, key, val):
        self.browser.log_debug("set header: %s=%s" % (key, val))
        with self.lock:
            if val is not None:
                self.header[key] = val
            elif key in self.header:
                del self.header[key]

    def get_cookies_str(self, url):
        """
        get_cookies_str() returns only the cookies applicable to given *url*
        """

        _, _, path = parse_url(url)

        path_cookies = {}
        for key, morsel in self.cookies.items():
            p = morsel["path"] if morsel["path"] else "/"
            if key not in path_cookies:
                path_cookies[key] = {}
            path_cookies[key][p] = morsel.value

        cookies = {}

        for key, paths in path_cookies.items():
            # sort by string length: ["/home/path", "/home", "/"]
            for p in sorted(paths.keys(), lambda x: len(x), reverse=True):
                if path.startswith(p):
                    cookies[key] = paths[p]
                    break

        return "; ".join("%s=%s" % (key, val) for key, val in cookies.items())

    def _get_redirected_request(self, req, response):
        new_req = None
        if req.status in [301, 302, 303]:
            new_req = req.duplicate()
            new_req.method = 'GET'
            new_req.params = None
            new_req.url = response.getheader('location')
        elif self.browser.js_redirects:
            page_actions = httputils.PageActionsExtractor(req.data)
            redirect = page_actions.get_action()
            if redirect:
                new_req = req.duplicate()
                new_req.method = redirect.method
                new_req.params = redirect.get_params()
                new_req.url = redirect.url
                new_req.header[
                    'Cookie'] = self.browser.browser_get_cookies_str(req.url)
            req.page_actions = page_actions

        if new_req:
            new_req.validator = None
            new_req.header['Cookie'] = self.browser.browser_get_cookies_str(
                req.url)
            if not new_req.url.startswith('http'):
                new_req.update_netloc(req.url)
            _, new_req.header['Host'], _ = parse_url(new_req.url)
            return new_req

        return None

    def _execute_page_request(self, pool, page, req, path_with_args):
        assert (req.method == 'POST'
                or not req.params)  # only POST request may have params

        with pool.borrow() as (conn, is_new):
            # conn.set_debuglevel(1)
            conn.request(req.method, path_with_args, req.params, req.header)
            self.header['Referer'] = req.url
            response = conn.getresponse()
            req.status = response.status
            req.data = response.read()

            self.browser.log_debug(" req %s HTTP response: %s, headers: %s" %
                                   (req.id, req.status, response.getheaders()))

            # handle new cookies
            for cookie in extract_cookies(response):
                if sys.version_info[0] < 3 and isinstance(cookie, unicode):
                    cookie = cookie.encode('ascii', 'ignore')
                self.cookies.load(cookie)

            # handle redirection
            new_req = self._get_redirected_request(req, response)
            if new_req:
                self.browser.log_debug(" req %s, redirect to %s" %
                                       (new_req.id, new_req.url))
                page.add_request(new_req)

                # FIXME:
                # 1. _get_netloc_data is private method of the browser.

                self.browser._get_netloc_data(
                    new_req.url).execute_page_request(page, new_req)
                req.status = new_req.status
                req.data = new_req.data

            if req.valid_statuses and req.status not in req.valid_statuses:
                raise BrowserExc(' req %s, %s %s status %d' (req.id,
                                                             req.method,
                                                             req.url,
                                                             req.status))

        if self.browser.validation:
            req.validate_response(req.data)

        req.complete()
        return

    def execute_page_request(self, page, req):

        scheme, netloc, path_with_args = parse_url(req.url, args=True)

        if scheme not in ("http", "https", "ftp"):
            msg = "Can't execute request on URL with unsupported scheme: %s, %s" % (
                scheme, netloc)
            self.browser.log_error(msg)
            raise BrowserExc(msg)

        req.start()

        pool = self._get_http_pool(scheme)
        # 10 - is technological retry count to handle closed keep-alive connections
        for loop in range(10):
            try:
                self._execute_page_request(pool, page, req, path_with_args)
                return

            except pool.temporary_errors as ex:
                raise  # aandreev
                # usually it means server has closed keep-alive connection due to timeout. lets retry
                self.browser.log_debug(
                    "HTTP Exception: %s %s: %s %s, connection closed? retrying"
                    % (req.method, req.url, type(ex), str(ex)))
                req.status = str(type(ex))
                time.sleep(0.1 * loop)

            except pool.fatal_errors as ex:
                req.status = str(type(ex))
                req.complete()
                self.browser.log_error(
                    "HTTP Exception: %s %s: %s %s" %
                    (req.method, req.url, type(ex), str(ex)))
                return

        req.complete()
        self.browser.log_error("HTTPException: %s %s, all retries failed" %
                               (req.method, req.url))

    def execute_page_request_parallel(self, page, reqs, parallel=8):
        if not len(reqs):
            return
        if len(reqs) == 1:
            self.execute_page_request(page, reqs[0])
            return

        def execute(arg):
            global data
            page, req = arg
            try:
                self.execute_page_request(page, req)
            except RuntimeError:
                import traceback
                self.browser.log_error("traceback:\n" + traceback.format_exc())
            return 0

        args = [(page, req) for req in reqs]

        if parallel > len(reqs):
            parallel = len(reqs)

        pool = ThreadPool(parallel)
        pool.map(execute, args)
        pool.close()
        pool.join()
Пример #32
0
def call_wsgi_app(wsgi_app, request, path_info):
    """
    Call the ``wsgi_app`` with ``request`` and return its response.
    
    :param wsgi_app: The WSGI application to be run.
    :type wsgi_app: callable
    :param request: The Django request.
    :type request: :class:`twod.wsgi.handler.TwodWSGIRequest`
    :param path_info: The ``PATH_INFO`` to be used by the WSGI application.
    :type path: :class:`basestring`
    :raises twod.wsgi.exc.ApplicationCallError: If ``path_info`` is not the
        last portion of the ``PATH_INFO`` in ``request``.
    :return: The response from the WSGI application, turned into a Django
        response.
    :rtype: :class:`twod.wsgi.TwodResponse`
    
    """
    new_request = request.copy()
    
    # Moving the portion of the path consumed by the current view, from the
    # PATH_INTO to the SCRIPT_NAME:
    if not request.path_info.endswith(path_info):
        raise ApplicationCallError("Path %s is not the last portion of the "
                                   "PATH_INFO in the original request (%s)"
                                   % (path_info, request.path_info))
    consumed_path = request.path_info[:-len(path_info)]
    new_request.path_info = path_info
    new_request.script_name = request.script_name + consumed_path
    
    # If the user has been authenticated in Django, log him in the WSGI app:
    if request.user.is_authenticated():
        new_request.remote_user = request.user.username
    
    # Cleaning the routing_args, if any. The application should have its own
    # arguments, without relying on any arguments from a parent application:
    if "wsgiorg.routing_args" in request.environ:
        del new_request.environ['wsgiorg.routing_args']
    # And the same for the WebOb ad-hoc attributes:
    if "webob.adhoc_attrs" in request.environ:
        del new_request.environ['webob.adhoc_attrs']
    
    # Calling the WSGI application and getting its response:
    (status, headers, body) = new_request.call_application(wsgi_app)
    
    # Turning its response into a Django response:
    cookies = SimpleCookie()
    django_response = TwodResponse(body, status=status)
    for (header, value) in headers:
        if header.upper() == "SET-COOKIE":
            if isinstance(value, unicode):
                # It can't be Unicode:
                value = value.encode("us-ascii")
            cookies.load(value)
        else:
            django_response[header] = value
    
    # Setting the cookies from Django:
    for (cookie_name, cookie) in cookies.items():
        cookie_attributes = {
            'key': cookie_name,
            'value': cookie.value,
            'max_age': cookie.get("max-age"),
            'expires': cookie.get("expires"),
            'path': cookie.get("path", "/"),
            'domain': cookie.get("domain"),
            }
        django_response.set_cookie(**cookie_attributes)
    return django_response
Пример #33
0
    def _extract_response(page, encoding='utf8'):
        history = []
        set_cookies = []
        res = None
        try:
            for i, url in enumerate(page['history']):
                resource = page['resources'].pop(0)
                while resource['request']['url'] != url:
                    resource = page['resources'].pop(0)

                if resource['error']:
                    return resource['error'], None

                request = resource['request']
                req = PreparedRequest()
                req.method = request['method'].encode(encoding)
                req.url = request['url'].encode(encoding)

                # Set Request Headers
                req.headers = CaseInsensitiveDict()
                for header in request['headers']:
                    req.headers[header['name'].encode(encoding)] = header['value'].encode(encoding)

                # Set Request Cookies
                req._cookies = RequestsCookieJar()
                if set_cookies:
                    if 'Cookie' not in req.headers:
                        req.headers['Cookie'] = ""
                    else:
                        set_cookies.insert(0, '')
                    req.headers['Cookie'] += "; ".join(set_cookies)

                if 'Cookie' in req.headers:
                    cookies = SimpleCookie()
                    cookies.load(req.headers['Cookie'])
                    for key, cookie in cookies.items():
                        req._cookies.set(key, cookie.value)

                req.body = request.get('postData', None)
                if req.body:
                    req.body = req.body.encode(encoding)

                response = resource['endReply'] or resource['startReply']
                res = Response()
                res.encoding = encoding
                res.url = response['url'].encode(encoding)
                res.status_code = response['status']
                for header in response['headers']:
                    res.headers[header['name'].encode(encoding)] = header['value'].encode(encoding)
                    if header['name'] == 'Set-Cookie':
                        set_cookies.append(res.headers[header['name']].rsplit(';', 1)[0])

                res.history = list(history)
                res.request = req

                history.append(res)

            res._content = re.sub(
                (
                    '<html><head></head><body>'
                    '<pre style="word-wrap: break-word; white-space: pre-wrap;">(.*?)</pre>'
                    '</body></html>'
                ),
                r'\1',
                page['content'],
                flags=re.DOTALL
            ).encode(encoding)

            return None, res
        except IndexError:
            return {'errorCode': -1,
                    'errorString': 'An existing connection was forcibly closed by the remote host'}, None
Пример #34
0
def call_wsgi_app(wsgi_app, request, path_info):
    """
    Call the ``wsgi_app`` with ``request`` and return its response.
    
    :param wsgi_app: The WSGI application to be run.
    :type wsgi_app: callable
    :param request: The Django request.
    :type request: :class:`twod.wsgi.handler.TwodWSGIRequest`
    :param path_info: The ``PATH_INFO`` to be used by the WSGI application.
    :type path: :class:`basestring`
    :raises twod.wsgi.exc.ApplicationCallError: If ``path_info`` is not the
        last portion of the ``PATH_INFO`` in ``request``.
    :return: The response from the WSGI application, turned into a Django
        response.
    :rtype: :class:`twod.wsgi.TwodResponse`
    
    """
    new_request = request.copy()

    # Moving the portion of the path consumed by the current view, from the
    # PATH_INTO to the SCRIPT_NAME:
    if not request.path_info.endswith(path_info):
        raise ApplicationCallError("Path %s is not the last portion of the "
                                   "PATH_INFO in the original request (%s)" %
                                   (path_info, request.path_info))
    consumed_path = request.path_info[:-len(path_info)]
    new_request.path_info = path_info
    new_request.script_name = request.script_name + consumed_path

    # If the user has been authenticated in Django, log him in the WSGI app:
    if request.user.is_authenticated():
        new_request.remote_user = request.user.username

    # Cleaning the routing_args, if any. The application should have its own
    # arguments, without relying on any arguments from a parent application:
    if "wsgiorg.routing_args" in request.environ:
        del new_request.environ['wsgiorg.routing_args']
    # And the same for the WebOb ad-hoc attributes:
    if "webob.adhoc_attrs" in request.environ:
        del new_request.environ['webob.adhoc_attrs']

    # Calling the WSGI application and getting its response:
    (status, headers, body) = new_request.call_application(wsgi_app)

    # Turning its response into a Django response:
    cookies = SimpleCookie()
    django_response = TwodResponse(body, status=status)
    for (header, value) in headers:
        if header.upper() == "SET-COOKIE":
            if isinstance(value, unicode):
                # It can't be Unicode:
                value = value.encode("us-ascii")
            cookies.load(value)
        else:
            django_response[header] = value

    # Setting the cookies from Django:
    for (cookie_name, cookie) in cookies.items():
        cookie_attributes = {
            'key': cookie_name,
            'value': cookie.value,
            'max_age': cookie.get("max-age"),
            'expires': cookie.get("expires"),
            'path': cookie.get("path", "/"),
            'domain': cookie.get("domain"),
        }
        django_response.set_cookie(**cookie_attributes)
    return django_response