def _get_indicator_data(self, alpha3_text, indicator):
query = "countries/{countries}/indicators/{indicator}{params}".format(
countries=alpha3_text,
indicator=indicator,
params="?format=json&per_page=5000", # lower limit
)
url = urllib.parse.urljoin(self.BASE_URL, query)
fetch_response = utils.fetch(url)
response_json = json.loads(fetch_response)
header = {}
indicator_data = []
if len(response_json) > 0:
header = response_json[0]
if len(response_json) > 1:
indicator_data = response_json[1]
self.progress["indicator_pages"] = header.get("pages", 1)
self.progress["current_page"] = 1
# loop through the rest of the pages if they exist
for page in range(2, header.get("pages", 1) + 1):
self.progress["current_page"] = page
page_url = "{url}&page={page}".format(url=url, page=page)
indicator_data += json.loads(utils.fetch(page_url))[1]
return indicator_data
def test_fetch_single(self, gettempdir, get):
"""Test single fetch request"""
gettempdir.return_value = self.TEST_TEMP_DIR
get().text = "dummy result"
result = utils.fetch("http://api.worldbank.org/indicators?format=json")
self.assertEqual("dummy result", result)
def get_countries(self):
"""Get a list of countries and regions.
This is a full list of countries and regions that can be used in the
indicator queries. In the list there are some non ISO codes such as WLD
for the whole world and other aggregate regions.
Note that this list does not include all countries (e.g. Israel),
but only countries that can be queried with the indicator API.
Returns:
list[dict]: A list of countries and aggregate regions.
"""
country_query = "countries" + self.GET_PARAMS
url = urllib.parse.urljoin(self.BASE_URL, country_query)
country_result = utils.fetch(url)
countries = json.loads(country_result)[1]
dict_to_text = ["region", "adminregion", "incomeLevel", "lendingType"]
for country in countries:
for key in dict_to_text:
country[key + "_text"] = "{value} ({id_})".format(
value=country.get(key, {}).get("value"),
id_=country.get(key, {}).get("id"),
)
return countries
def test_fetch(self, gettempdir, getmtime, get):
"""Test fetching from cache."""
gettempdir.return_value = self.TEST_TEMP_DIR
getmtime.return_value = int(time.time())
get().text = "dummy result"
res_1 = utils.fetch("http://api.worldbank.org/indicators?format=json")
self.assertEqual("dummy result", res_1)
get().text = "Updated dummy Result!"
res_2 = utils.fetch("http://google.com")
self.assertEqual("Updated dummy Result!", res_2)
res_1 = utils.fetch("http://api.worldbank.org/indicators?format=json")
self.assertEqual("dummy result", res_1)
getmtime.return_value = int(time.time()) - utils.CACHE_TIME - 10
res_1 = utils.fetch("http://api.worldbank.org/indicators?format=json")
self.assertEqual("Updated dummy Result!", res_1)
get().text = "Third dummy result."
res_2 = utils.fetch("http://google.com", use_cache=False)
self.assertEqual("Third dummy result.", res_2)
def get_indicators(self, filter_="Common"):
"""Get a list of indicators.
Args:
filter_ (str): Common or Featured. Leave empty for all indicators.
Returns:
list[dict]: A list of queryable indicators.
"""
country_query = "indicators" + self.GET_PARAMS
url = urllib.parse.urljoin(self.BASE_URL, country_query)
indicators = json.loads(utils.fetch(url))[1]
if filter_:
return self._filter_indicators(indicators, filter_)
return indicators
def get_instrumental(self, locations, data_types=None, intervals=None):
"""Get historical data for temperature or precipitation.
data_type (str): ``pr`` for precipitation, or ``tas`` for temperature.
If None is specified, both values will be returned.
interval (str): Either ``year``, ``month`` or ``decade``.
locations (list[str]): A list of API location codes - basin id numbers,
or ISO alpha-2 or alpha-3
flat_results (bool): If false, the result will be returned as a nested
dict, otherwise the result will be a single level dict with a
tuple key and normal value.
Returns:
dict: Nested dict with location on the first level, data type on
the second, interval on the third and the appropriate data.
"""
if not data_types:
data_types = self.INSTRUMENTAL_TYPES
if not intervals:
intervals = self._default_intervals
api_responses = defaultdict(lambda: defaultdict(dict))
parameters = list(product(locations, data_types, intervals))
self.progress["pages"] = len(parameters)
self.progress["current_page"] = 0
for location, data_type, interval in parameters:
self.progress["current_page"] += 1
loc_type, location = self._get_location(location)
query = self.INSTRUMENTAL_QUERY.format(
loc_type=loc_type,
data_type=data_type,
interval=interval,
location=location,
)
url = self.BASE_URL + query
api_responses[location][data_type][interval] = {
"url": url,
"response": json.loads(utils.fetch(url))
}
return self._dataset_class(api_responses)
def get_instrumental(self, locations, data_types=None, intervals=None):
"""Get historical data for temperature or precipitation.
data_type (str): ``pr`` for precipitation, or ``tas`` for temperature.
If None is specified, both values will be returned.
interval (str): Either ``year``, ``month`` or ``decade``.
locations (list[str]): A list of API location codes - basin id numbers,
or ISO alpha-2 or alpha-3
flat_results (bool): If false, the result will be returned as a nested
dict, otherwise the result will be a single level dict with a
tuple key and normal value.
Returns:
dict: Nested dict with location on the first level, data type on
the second, interval on the third and the appropriate data.
"""
if not data_types:
data_types = self.INSTRUMENTAL_TYPES
if not intervals:
intervals = self._default_intervals
api_responses = defaultdict(lambda: defaultdict(dict))
parameters = list(product(locations, data_types, intervals))
self.progress["pages"] = len(parameters)
self.progress["current_page"] = 0
for location, data_type, interval in parameters:
self.progress["current_page"] += 1
loc_type, location = self._get_location(location)
query = self.INSTRUMENTAL_QUERY.format(
loc_type=loc_type,
data_type=data_type,
interval=interval,
location=location,
)
url = self.BASE_URL + query
api_responses[location][data_type][interval] = {
"url": url,
"response": json.loads(utils.fetch(url))
}
return self._dataset_class(api_responses)