def geturlcon(self, url, data=None, headers={}, timeout=20, retries=5):
try:
result = super(ChinawutongHttpClient, self).geturlcon(url, data, headers, timeout, retries).decode('gb2312',
'ignore')
#self.saveCookie()
time.sleep(5 + 2 * random.random())
logger.warn(result)
return result
except:
return None
def geturlcon(self, url, data=None, headers={}, timeout=20, retries=5):
logger.info(url)
tries = 0
while tries < retries:
try:
request = urllib2.Request(url, data=self.getPostData(data), headers=headers)
url = urllib2.urlopen(request, timeout=timeout)
time.sleep(1)
#page = url.read().decode('utf8', 'ignore')
page = url.read()
logger.warn(page)
return page
except Exception, e:
print traceback.format_exc()
tries += 1
time.sleep(5 * random.random() + 5)