async def get_state_data(self):
promises = await asyncio.gather(
self.DATA_SERVICE.get_data(self.ENDPOINT),
self.LOCATION_SERVICE.get_state_data(),
)
results_by_county, last_updated = promises[0]
state_data = promises[1]
# Aggregate results on a per state basis
state_results = {}
for result in results_by_county:
key = tuple(result.id.split("@")[:2])
if key not in state_results:
properties_for_state = (state_data[key] if key in state_data
else LocationProperties())
state_results[key] = JhuLocation(
id=Functions.to_location_id(key),
uid=properties_for_state.uid,
iso2=properties_for_state.iso2,
iso3=properties_for_state.iso3,
code3=properties_for_state.code3,
fips=properties_for_state.fips,
county=properties_for_state.admin2,
state=result.state,
country=result.country,
latitude=properties_for_state.coordinates.latitude,
longitude=properties_for_state.coordinates.longitude,
last_updated=last_updated,
timelines={
"confirmed": {},
"deaths": {}
},
latest=None,
)
jhu_location = state_results[key]
for confirmed_date, count in result.timelines[
"confirmed"].category.items():
value = jhu_location.timelines["confirmed"].get(
confirmed_date, 0)
jhu_location.timelines["confirmed"][
confirmed_date] = value + count
for deaths_date, count in result.timelines[
"deaths"].category.items():
value = jhu_location.timelines["deaths"].get(deaths_date, 0)
jhu_location.timelines["deaths"][deaths_date] = value + count
# Remap dicts to Category
for _, state in state_results.items():
state.timelines["confirmed"] = Category(
state.timelines["confirmed"])
state.timelines["deaths"] = Category(state.timelines["deaths"])
state.latest = Statistics(
state.timelines["confirmed"].latest,
state.timelines["deaths"].latest).to_dict()
return state_results.values(), last_updated
def test__empty_category__success():
# Arrange
category = Category()
# Act and Assert
assert category.category is not None
assert len(category.category) == 0
assert category.latest == 0
expected_dict = {"latest": 0, "history": OrderedDict()}
assert category.to_dict() == expected_dict
async def get_country_data(self) -> (List[JhuLocation], str):
"""Notes: Function currently designed only for US data
"""
promises = await asyncio.gather(
self.DATA_SERVICE.get_data(self.ENDPOINT), )
results_by_county, last_updated = promises[0]
location_properties = JhuLocation(
id=Functions.to_location_id(("US", )),
uid="840",
iso2="US",
iso3="USA",
code3="USA",
fips="",
county="",
state="",
country="US",
latitude="37.0902", # TODO: Do not hardcode
longitude="-95.7129",
last_updated=last_updated,
timelines={
"confirmed": {},
"deaths": {}
},
latest=None,
)
for result in results_by_county:
for confirmed_date, count in result.timelines[
"confirmed"].category.items():
value = location_properties.timelines["confirmed"].get(
confirmed_date, 0)
location_properties.timelines["confirmed"][confirmed_date] = (
value + count)
for deaths_date, count in result.timelines[
"deaths"].category.items():
value = location_properties.timelines["deaths"].get(
deaths_date, 0)
location_properties.timelines["deaths"][
deaths_date] = value + count
location_properties.timelines["confirmed"] = Category(
location_properties.timelines["confirmed"])
location_properties.timelines["deaths"] = Category(
location_properties.timelines["deaths"])
location_properties.latest = Statistics(
location_properties.timelines["confirmed"].latest,
location_properties.timelines["deaths"].latest,
).to_dict()
return [location_properties], last_updated
async def _build_results_cached(
self,
grouped_locations: dict,
data_type: str,
last_updated: str,
from_cache: bool,
):
"""Given a dictionary of data aggregated by location_id, build each entry into an NytLocation object and build a serialized version of it for caching.
Arguments:
grouped_locations {dict} -- dictionary of data aggregated by location_id.
data_type {str} -- category of data we retrieved from Nyt (country, state, county)
last_updated {str} -- date the information was last updated.
from_cache {bool} -- if the dictionary of aggregated data received was from cache.
"""
from backend.utils.containers import Container
locations = []
to_serialize = {"locations": {}}
for location_id, events in grouped_locations.items():
confirmed_map = events["confirmed"]
deaths_map = events["deaths"]
confirmed = Category(confirmed_map)
deaths = Category(deaths_map)
locations.append(
NytLocation(
id=location_id,
country=events["country"],
county=events["county"],
state=events["state"],
fips=events["fips"],
timelines={
"confirmed": confirmed,
"deaths": deaths,
},
last_updated=last_updated,
latest=Statistics(confirmed=confirmed.latest,
deaths=deaths.latest).to_dict(),
))
# Transform to store in cache
if not from_cache:
to_serialize["locations"][location_id] = self._serialize_entry(
location_id, grouped_locations, confirmed_map, deaths_map)
if not from_cache:
await Container.cache().set_item(f"nyt_data_{data_type}",
to_serialize)
return locations
def test__given_valid_location__to_dict__success():
# Arrange
location = Location(**TestBase.VALID_LOCATION)
# Act & Assert
assert location.to_dict() == {
"id": TestBase.VALID_LOCATION["id"],
"country": TestBase.VALID_LOCATION["country"],
"last_updated": TestBase.VALID_LOCATION["last_updated"],
"latest": TestBase.VALID_LOCATION["latest"],
}
assert location.to_dict(True) == {
"id": TestBase.VALID_LOCATION["id"],
"country": TestBase.VALID_LOCATION["country"],
"last_updated": TestBase.VALID_LOCATION["last_updated"],
"latest": TestBase.VALID_LOCATION["latest"],
"timelines": {
"confirmed": Category({
"03-24-20": 5
}).to_dict(),
"deaths": Category({
"03-24-20": 1
}).to_dict(),
},
}
assert location.to_dict(True, True) == {
"id": TestBase.VALID_LOCATION["id"],
"country": TestBase.VALID_LOCATION["country"],
"last_updated": TestBase.VALID_LOCATION["last_updated"],
"latest": TestBase.VALID_LOCATION["latest"],
"timelines": {
"confirmed": Category({
"03-24-20": 5
}).to_dict(),
"deaths": Category({
"03-24-20": 1
}).to_dict(),
},
"properties": None,
}
def test__valid_category__success():
# Arrange
category = Category(TEST_CATEGORY_DATA)
# Act and Assert
assert category.category is not None
assert len(category.category) == 6
assert category.latest == 158
expected_dict = {
"latest":
158,
"history":
OrderedDict([
("2020-02-01", 5),
("2020-02-02", 6),
("2020-02-03", 21),
("2020-02-04", 84),
("2020-02-05", 120),
("2020-02-06", 158),
]),
}
assert category.to_dict() == expected_dict
async def get_data(
self, endpoint: str, data_type: str = ""
) -> (List[JhuLocation], str):
"""Method that retrieves data from JHU CSSEGSI.
Arguments:
endpoint {str} -- string that represents endpoint to get data from.
data_type {str} -- string that represents type of data being fetched. Used as key for cache.
Returns:
Location[], str -- returns list of location stats and the last updated date.
"""
_start = time.time() * 1000.0
promises = await asyncio.gather(
self._fetch_csv_data(endpoint, "confirmed"),
self._fetch_csv_data(endpoint, "deaths"),
)
_end = time.time() * 1000.0
logger.info(f"Elapsed _fetch_csv_data for all stats {str(_end-_start)}ms")
_start = time.time() * 1000.0
tagged_promises = self._tag_promised_results(
["confirmed", "deaths"], promises
) # [("confirmed", ...), ...]
location_result = await self._zip_results(
tagged_promises
) # Store the final map of datapoints { "Locations": {}, "first_date": {} }
_end = time.time() * 1000.0
logger.info(f"Elapsed _zip_results for all stats {str(_end-_start)}ms")
locations = []
last_updated = Functions.get_formatted_date()
for location_id, events, in location_result.items():
confirmed_map = events["confirmed"]
deaths_map = events["deaths"]
confirmed = Category(confirmed_map)
deaths = Category(deaths_map)
locations.append(
JhuLocation(
id=location_id,
uid=events["uid"],
iso2=events["iso2"],
iso3=events["iso3"],
code3=events["code3"],
fips=events["FIPS"],
county=events["Admin2"],
state=events["Province_State"],
country=events["Country_Region"],
latitude=events["Lat"],
longitude=events["Long_"],
last_updated=last_updated,
timelines={"confirmed": confirmed, "deaths": deaths},
latest=Statistics(
confirmed=confirmed.latest, deaths=deaths.latest
).to_dict(),
)
)
logger.info("Finished transforming JHU results.")
return locations, last_updated