def fetch_json_api(url, use_cache_on_failure=False):
LOG.debug("Fetching %s...", url)
num_tries = 0
while True:
resp_json = Fetcher._try_fetching_resp(url)
LOG.debug("Response from %s: %s", url, resp_json)
if resp_json:
Fetcher._cache[url] = resp_json
return resp_json
elif use_cache_on_failure and url in Fetcher._cache:
LOG.warn(
"Bad response from '%s\'! Falling back to cache...\033[0m\n",
url)
return Fetcher._cache[url]
num_tries += 1
sleep_time = min(2**num_tries - 1, Fetcher._MAX_WAIT)
LOG.warning(
"Bad response from '%s\'! Backing off for %i second(s)...",
url, sleep_time)
sleep(sleep_time)
def _check_file_descriptors():
warn_at = 20
num_fds = psutil.Process().num_fds()
if num_fds > warn_at:
LOG.warn("%i file descriptors open!", num_fds)