def open(self,url):
"""Handles all errors by waiting 2 seconds and trying again."""
while True:
my_time = time()
if self.prev_time:
# If something messes with the system clock, this assert might fire:
assert(my_time > self.prev_time)
sleep(self.WAIT_TIME - min((my_time - self.prev_time), self.WAIT_TIME))
self.prev_time = my_time
try:
out = www.open_http(url)
return out
except:
print "HTTP failed"
sleep(2)
def open(self,url,verbose=False):
"""Handles all errors by crashing out."""
while True:
my_time = time()
if self.prev_time:
# If something messes with the system clock, this assert might fire:
assert(my_time > self.prev_time)
sleep(self.WAIT_TIME - min((my_time - self.prev_time), self.WAIT_TIME))
self.prev_time = my_time
try:
res = www.open_http(url,self.user_agent,True)
if verbose:
return ''.join(res.readlines()), res.geturl(), res.info()
else:
return ''.join(res.readlines()), res.geturl()
except:
print "HTTP failed"
print url
raise
