def read_url(url): count = 5 wait = 5 retry = 0 ex = None while retry < count: try: res = urllib.urlopen(url).read() return res except Exception as e: logger.warn("request for url: %s, failed."%url) retry = retry + 1 time.sleep(wait) ex = e raise ex
def read_json(url, func = None): count = 5 retry = 0 ex = None urlresponse = None while retry < count: try: urlresponse = Net.read_url(url) if func != None: urlresponse = func(urlresponse) res = simplejson.loads(urlresponse) return res except Exception as e: logger.warn("parse json for url: %s, failed. jsonstr: %s."%(url, urlresponse)) retry = retry + 1 ex = e raise ex