def open(self,url): """Handles all errors by waiting 2 seconds and trying again.""" while True: my_time = time() if self.prev_time: # If something messes with the system clock, this assert might fire: assert(my_time > self.prev_time) sleep(self.WAIT_TIME - min((my_time - self.prev_time), self.WAIT_TIME)) self.prev_time = my_time try: out = www.open_http(url) return out except: print "HTTP failed" sleep(2)
def open(self,url,verbose=False): """Handles all errors by crashing out.""" while True: my_time = time() if self.prev_time: # If something messes with the system clock, this assert might fire: assert(my_time > self.prev_time) sleep(self.WAIT_TIME - min((my_time - self.prev_time), self.WAIT_TIME)) self.prev_time = my_time try: res = www.open_http(url,self.user_agent,True) if verbose: return ''.join(res.readlines()), res.geturl(), res.info() else: return ''.join(res.readlines()), res.geturl() except: print "HTTP failed" print url raise