Пример #1
0
def post_and_get_content(url, param=None):
    cookie = MySimpleCookie.load_from_session_or_new(url)
    #===========================================================================
    # is_Google_login = url.startswith('https://www.google.com/accounts/ServiceLoginAuth')
    # if is_Google_login:
    #    from GoogleLogin import GoogleLogin
    #    auth = GoogleLogin(param['Email'],param['Passwd'],param['continue'],cookiejar )
    #    cookiejar = auth.cookiejar
    #===========================================================================

    #p = '127.0.0.1:5865'
    #opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar),ConnectHTTPSHandler(proxy=p))
    from URLOpener import URLOpener

    #opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
    opener = URLOpener(cookie)

    #===========================================================================
    # opener.addheaders = [('User-agent',  'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)')]
    # urllib2.install_opener(opener)
    # #if is_Google_login:
    # #    req=urllib2.Request(param['continue'])
    # #else:
    # req=urllib2.Request(url,param and urllib.urlencode(param) or None)
    #
    # resp=urllib2.urlopen(req)
    #===========================================================================

    resp = opener.open(url, param and urllib.urlencode(param) or None)

    content = resp.content  #read()

    #save
    cookie = opener.cookie
    cookie.save_to_memcache()

    #headers = dict([ ( (x[:x.index(':')]).strip(), (x[x.index(':')+1:]).strip()) for x in  str(resp.info()).split('\n') if (x.strip()!='' and x.index(':')>=0) ]) # resp.info()#.headers # resp.headers #iteritems
    #status_code = '200' # resp.info().status #_code# '200' #resp.status_code
    return {
        'content': content,
        'headers': resp.headers,
        'status_code': resp.status_code
    }
Пример #2
0
def post_and_get_content(url,param=None):     
    cookie    = MySimpleCookie.load_from_session_or_new(url)    
    #===========================================================================
    # is_Google_login = url.startswith('https://www.google.com/accounts/ServiceLoginAuth')
    # if is_Google_login:
    #    from GoogleLogin import GoogleLogin
    #    auth = GoogleLogin(param['Email'],param['Passwd'],param['continue'],cookiejar )
    #    cookiejar = auth.cookiejar
    #===========================================================================
        
    #p = '127.0.0.1:5865'
    #opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar),ConnectHTTPSHandler(proxy=p)) 
    from URLOpener import URLOpener
        
    #opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) 
    opener = URLOpener(cookie)
    
    #===========================================================================
    # opener.addheaders = [('User-agent',  'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)')]
    # urllib2.install_opener(opener) 
    # #if is_Google_login:
    # #    req=urllib2.Request(param['continue'])
    # #else:
    # req=urllib2.Request(url,param and urllib.urlencode(param) or None)
    #     
    # resp=urllib2.urlopen(req)
    #===========================================================================
    
    resp = opener.open(url,param and urllib.urlencode(param) or None)

    
    content =  resp.content#read()
    
    #save    
    cookie = opener.cookie
    cookie.save_to_memcache()
    
    #headers = dict([ ( (x[:x.index(':')]).strip(), (x[x.index(':')+1:]).strip()) for x in  str(resp.info()).split('\n') if (x.strip()!='' and x.index(':')>=0) ]) # resp.info()#.headers # resp.headers #iteritems
    #status_code = '200' # resp.info().status #_code# '200' #resp.status_code
    return {'content':content,'headers':resp.headers,'status_code':resp.status_code}
Пример #3
0
 def open(self, url, data = None,headers={}):
     self.cookie =MySimpleCookie.load_from_session_or_new(url)
     
     if data is None:
         method = urlfetch.GET
     else:
         method = urlfetch.POST
     base_url = url
     while url is not None:
         headers_ = self._getHeaders(self.cookie)
         if headers is not None:
             headers_.update(headers)
         logging.error('fetching %s'%url)
         response = urlfetch.fetch(url=url,
                                   payload=data,
                                   method=method,
                                   headers=headers_,
                                   allow_truncated=False,
                                   follow_redirects=False,
                                   deadline=10
                 )
         response.lasturl = url
         data = None # Next request will be a get, so no need to send the data again.
         method = urlfetch.GET
         
         self.cookie.load(response.headers.get('set-cookie', ''))  #Load the cookies from the response
         #===================================================================
         # if Cookie.SimpleCookie().load(response.headers.get('set-cookie', '')) is not None:
         #    for item in Cookie.SimpleCookie().load(response.headers.get('set-cookie', '')): 
         #        self.cookie[item.key] = item.value #@IndentOk
         # 
         #===================================================================
         url = response.headers.get('location')
         if not url is None and  not url.startswith('http'):
             url = urlparse.urljoin(base_url, url)
     
     self.cookie.save_to_memcache()
     return response
Пример #4
0
    def open(self, url, data=None, headers={}):
        self.cookie = MySimpleCookie.load_from_session_or_new(url)

        if data is None:
            method = urlfetch.GET
        else:
            method = urlfetch.POST
        base_url = url
        while url is not None:
            headers_ = self._getHeaders(self.cookie)
            if headers is not None:
                headers_.update(headers)
            logging.error('fetching %s' % url)
            response = urlfetch.fetch(url=url,
                                      payload=data,
                                      method=method,
                                      headers=headers_,
                                      allow_truncated=False,
                                      follow_redirects=False,
                                      deadline=10)
            response.lasturl = url
            data = None  # Next request will be a get, so no need to send the data again.
            method = urlfetch.GET

            self.cookie.load(response.headers.get(
                'set-cookie', ''))  #Load the cookies from the response
            #===================================================================
            # if Cookie.SimpleCookie().load(response.headers.get('set-cookie', '')) is not None:
            #    for item in Cookie.SimpleCookie().load(response.headers.get('set-cookie', '')):
            #        self.cookie[item.key] = item.value #@IndentOk
            #
            #===================================================================
            url = response.headers.get('location')
            if not url is None and not url.startswith('http'):
                url = urlparse.urljoin(base_url, url)

        self.cookie.save_to_memcache()
        return response