def post_and_get_content(url, param=None):
cookie = MySimpleCookie.load_from_session_or_new(url)
#===========================================================================
# is_Google_login = url.startswith('https://www.google.com/accounts/ServiceLoginAuth')
# if is_Google_login:
# from GoogleLogin import GoogleLogin
# auth = GoogleLogin(param['Email'],param['Passwd'],param['continue'],cookiejar )
# cookiejar = auth.cookiejar
#===========================================================================
#p = '127.0.0.1:5865'
#opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar),ConnectHTTPSHandler(proxy=p))
from URLOpener import URLOpener
#opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
opener = URLOpener(cookie)
#===========================================================================
# opener.addheaders = [('User-agent', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)')]
# urllib2.install_opener(opener)
# #if is_Google_login:
# # req=urllib2.Request(param['continue'])
# #else:
# req=urllib2.Request(url,param and urllib.urlencode(param) or None)
#
# resp=urllib2.urlopen(req)
#===========================================================================
resp = opener.open(url, param and urllib.urlencode(param) or None)
content = resp.content #read()
#save
cookie = opener.cookie
cookie.save_to_memcache()
#headers = dict([ ( (x[:x.index(':')]).strip(), (x[x.index(':')+1:]).strip()) for x in str(resp.info()).split('\n') if (x.strip()!='' and x.index(':')>=0) ]) # resp.info()#.headers # resp.headers #iteritems
#status_code = '200' # resp.info().status #_code# '200' #resp.status_code
return {
'content': content,
'headers': resp.headers,
'status_code': resp.status_code
}
def post_and_get_content(url,param=None):
cookie = MySimpleCookie.load_from_session_or_new(url)
#===========================================================================
# is_Google_login = url.startswith('https://www.google.com/accounts/ServiceLoginAuth')
# if is_Google_login:
# from GoogleLogin import GoogleLogin
# auth = GoogleLogin(param['Email'],param['Passwd'],param['continue'],cookiejar )
# cookiejar = auth.cookiejar
#===========================================================================
#p = '127.0.0.1:5865'
#opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar),ConnectHTTPSHandler(proxy=p))
from URLOpener import URLOpener
#opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
opener = URLOpener(cookie)
#===========================================================================
# opener.addheaders = [('User-agent', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)')]
# urllib2.install_opener(opener)
# #if is_Google_login:
# # req=urllib2.Request(param['continue'])
# #else:
# req=urllib2.Request(url,param and urllib.urlencode(param) or None)
#
# resp=urllib2.urlopen(req)
#===========================================================================
resp = opener.open(url,param and urllib.urlencode(param) or None)
content = resp.content#read()
#save
cookie = opener.cookie
cookie.save_to_memcache()
#headers = dict([ ( (x[:x.index(':')]).strip(), (x[x.index(':')+1:]).strip()) for x in str(resp.info()).split('\n') if (x.strip()!='' and x.index(':')>=0) ]) # resp.info()#.headers # resp.headers #iteritems
#status_code = '200' # resp.info().status #_code# '200' #resp.status_code
return {'content':content,'headers':resp.headers,'status_code':resp.status_code}
def open(self, url, data = None,headers={}):
self.cookie =MySimpleCookie.load_from_session_or_new(url)
if data is None:
method = urlfetch.GET
else:
method = urlfetch.POST
base_url = url
while url is not None:
headers_ = self._getHeaders(self.cookie)
if headers is not None:
headers_.update(headers)
logging.error('fetching %s'%url)
response = urlfetch.fetch(url=url,
payload=data,
method=method,
headers=headers_,
allow_truncated=False,
follow_redirects=False,
deadline=10
)
response.lasturl = url
data = None # Next request will be a get, so no need to send the data again.
method = urlfetch.GET
self.cookie.load(response.headers.get('set-cookie', '')) #Load the cookies from the response
#===================================================================
# if Cookie.SimpleCookie().load(response.headers.get('set-cookie', '')) is not None:
# for item in Cookie.SimpleCookie().load(response.headers.get('set-cookie', '')):
# self.cookie[item.key] = item.value #@IndentOk
#
#===================================================================
url = response.headers.get('location')
if not url is None and not url.startswith('http'):
url = urlparse.urljoin(base_url, url)
self.cookie.save_to_memcache()
return response
def open(self, url, data=None, headers={}):
self.cookie = MySimpleCookie.load_from_session_or_new(url)
if data is None:
method = urlfetch.GET
else:
method = urlfetch.POST
base_url = url
while url is not None:
headers_ = self._getHeaders(self.cookie)
if headers is not None:
headers_.update(headers)
logging.error('fetching %s' % url)
response = urlfetch.fetch(url=url,
payload=data,
method=method,
headers=headers_,
allow_truncated=False,
follow_redirects=False,
deadline=10)
response.lasturl = url
data = None # Next request will be a get, so no need to send the data again.
method = urlfetch.GET
self.cookie.load(response.headers.get(
'set-cookie', '')) #Load the cookies from the response
#===================================================================
# if Cookie.SimpleCookie().load(response.headers.get('set-cookie', '')) is not None:
# for item in Cookie.SimpleCookie().load(response.headers.get('set-cookie', '')):
# self.cookie[item.key] = item.value #@IndentOk
#
#===================================================================
url = response.headers.get('location')
if not url is None and not url.startswith('http'):
url = urlparse.urljoin(base_url, url)
self.cookie.save_to_memcache()
return response