def covid19_webapp(instance_path=Path(__file__).parent, **kwargs):
logger.debug(f"Setting instance path to {instance_path}")
app = Flask(__name__, instance_path=instance_path)
with open(Path(__file__).parent / 'static' / 'pivotui_template.html',
'r') as html_file:
TEMPLATE = html_file.read()
async def pivot_ui(path, **kwargs) -> bytes:
df = await CovidData(cache_path=path).get_data()
csv = df.to_csv(encoding='utf8')
pickle_file = Path(path) / 'covid_weather_data.pkl'
modified_time = ctime(
getmtime(pickle_file)) if pickle_file.exists() else ctime(time())
if hasattr(csv, 'decode'):
csv = csv.decode('utf8')
return (TEMPLATE % dict(csv=csv,
modified_time=modified_time,
kwargs=json.dumps(kwargs))).encode()
@app.route('/')
def home():
loop = asyncio.new_event_loop()
html = loop.run_until_complete(
pivot_ui(path=app.instance_path, **kwargs))
return html.decode()
return app
async def _get_forecast_response(url: str, session: aiohttp.ClientSession, row: pd.Series) -> pd.Series:
async with session.get(url) as resp:
json = await resp.json()
log.debug(f"Processing json response {json}")
json = json['currently']
json['Lat'] = row['Lat']
json['Long'] = row['Long']
json['Date'] = row['Date']
return pd.Series(json)
async def _get_weather_from_forecast(self, df: pd.DataFrame) -> pd.DataFrame:
tasks = []
async with aiohttp.ClientSession() as session:
for index, row in df.iterrows():
log.debug(f"Adding async fetch task for Date:{row['Date']} Lat:{row['Lat']} Long:{row['Long']}")
task = asyncio.ensure_future(self._get_forecast_response(await self._forecast_url(row),
session,
row))
tasks.append(task)
log.debug(f"Executing {len(tasks)} async tasks to fetch weather data")
weather = pd.DataFrame(await asyncio.gather(*tasks))
return weather
async def _get_weather(self, covid_data: pd.DataFrame) -> pd.DataFrame:
if not self.api_key:
return pd.DataFrame()
pickle_file = self.cache_path / "weather.pkl"
if pickle_file.exists():
log.debug("Reading data from weather cache file")
weather_data = pd.read_pickle(pickle_file)
diff = covid_data[['Date', 'Lat', 'Long']].merge(
weather_data[['Date', 'Lat', 'Long']],
how='outer', indicator=True
).loc[lambda x: x['_merge'] == 'left_only'].drop('_merge', axis=1)
if not diff.empty:
weather_data = weather_data.append(await self._get_weather_from_forecast(diff))
else:
log.debug("Nothing new to fetch")
self.is_diff = False
else:
log.debug("Creating new pickle file")
weather_data = await self._get_weather_from_forecast(covid_data)
if self.is_diff:
log.debug(f"Writing {len(weather_data)} records to pickle file")
weather_data.to_pickle(pickle_file)
return weather_data
async def _fetch_covid19_data() -> pd.DataFrame:
log.debug("Fetching data from github covid dataset")
return pd.read_csv(
"https://raw.githubusercontent.com/datasets/covid-19/master/data/time-series-19-covid-combined.csv")
def __init__(self, cache_path=Path(__file__).parent):
self.api_key = environ.get("DARKSKY_KEY", None)
log.debug(f"Darksky api key: {self.api_key}")
self.cache_path = cache_path
self.is_diff = True
getmtime(pickle_file)) if pickle_file.exists() else ctime(time())
if hasattr(csv, 'decode'):
csv = csv.decode('utf8')
return (TEMPLATE % dict(csv=csv,
modified_time=modified_time,
kwargs=json.dumps(kwargs))).encode()
@app.route('/')
def home():
loop = asyncio.new_event_loop()
html = loop.run_until_complete(
pivot_ui(path=app.instance_path, **kwargs))
return html.decode()
return app
if __name__ == '__main__':
instance_path = Path(__file__).parent
if (instance_path / '.env').exists():
logger.debug(f"Loading {instance_path / '.env'}")
load_dotenv(instance_path / '.env')
app = covid19_webapp(instance_path=instance_path,
rows=["Country/Region"],
cols=["Date"],
aggregatorName="Maximum",
vals=["Deaths"],
rendererName="Line Chart")
app.run(host="0.0.0.0", port=8000, load_dotenv=True)