def get_json_dict_raw(url, cookies, proxy=False, times=1):
if exist(url):
return fetch(url)
if times > RETRY_TIMES:
log.error(
'Timeout for {} beyond the maximum({}) retry times. SKIP!'.format(
url, RETRY_TIMES))
return None
timer.sleep_awhile()
try:
if proxy and proxies != {}:
return requests.get(url,
headers=headers,
cookies=cookies,
timeout=5,
proxies=proxies).text
return requests.get(url, headers=headers, cookies=cookies,
timeout=5).text
except Timeout:
log.warn("timeout for {}. Try again.".format(url))
except Exception as e:
log.error("unknown error for {}. Try again. Error string: {}".format(
url, e))
log.error(traceback.format_exc())
data = get_json_dict_raw(url, cookies, proxy, times + 1)
return data
def get_json_dict(url, cookies = {}, proxy = False, times = 1, is_steam_request = 0):
if exist(url):
return json.loads(fetch(url))
json_data = get_json_dict_raw(url, cookies, proxy, times, is_steam_request)
if json_data is None:
return None
else:
# can not store None
store(url, json_data)
return json.loads(json_data)
def get_json_dict(url, cookies, proxy=False, times=1):
if exist(url):
return json.loads(fetch(url))
json_data = get_json_dict_raw(url, cookies, proxy, times)
store(url, json_data)
return json.loads(json_data)