def covid19_webapp(instance_path=Path(__file__).parent, **kwargs): logger.debug(f"Setting instance path to {instance_path}") app = Flask(__name__, instance_path=instance_path) with open(Path(__file__).parent / 'static' / 'pivotui_template.html', 'r') as html_file: TEMPLATE = html_file.read() async def pivot_ui(path, **kwargs) -> bytes: df = await CovidData(cache_path=path).get_data() csv = df.to_csv(encoding='utf8') pickle_file = Path(path) / 'covid_weather_data.pkl' modified_time = ctime( getmtime(pickle_file)) if pickle_file.exists() else ctime(time()) if hasattr(csv, 'decode'): csv = csv.decode('utf8') return (TEMPLATE % dict(csv=csv, modified_time=modified_time, kwargs=json.dumps(kwargs))).encode() @app.route('/') def home(): loop = asyncio.new_event_loop() html = loop.run_until_complete( pivot_ui(path=app.instance_path, **kwargs)) return html.decode() return app
async def _get_forecast_response(url: str, session: aiohttp.ClientSession, row: pd.Series) -> pd.Series: async with session.get(url) as resp: json = await resp.json() log.debug(f"Processing json response {json}") json = json['currently'] json['Lat'] = row['Lat'] json['Long'] = row['Long'] json['Date'] = row['Date'] return pd.Series(json)
async def _get_weather_from_forecast(self, df: pd.DataFrame) -> pd.DataFrame: tasks = [] async with aiohttp.ClientSession() as session: for index, row in df.iterrows(): log.debug(f"Adding async fetch task for Date:{row['Date']} Lat:{row['Lat']} Long:{row['Long']}") task = asyncio.ensure_future(self._get_forecast_response(await self._forecast_url(row), session, row)) tasks.append(task) log.debug(f"Executing {len(tasks)} async tasks to fetch weather data") weather = pd.DataFrame(await asyncio.gather(*tasks)) return weather
async def _get_weather(self, covid_data: pd.DataFrame) -> pd.DataFrame: if not self.api_key: return pd.DataFrame() pickle_file = self.cache_path / "weather.pkl" if pickle_file.exists(): log.debug("Reading data from weather cache file") weather_data = pd.read_pickle(pickle_file) diff = covid_data[['Date', 'Lat', 'Long']].merge( weather_data[['Date', 'Lat', 'Long']], how='outer', indicator=True ).loc[lambda x: x['_merge'] == 'left_only'].drop('_merge', axis=1) if not diff.empty: weather_data = weather_data.append(await self._get_weather_from_forecast(diff)) else: log.debug("Nothing new to fetch") self.is_diff = False else: log.debug("Creating new pickle file") weather_data = await self._get_weather_from_forecast(covid_data) if self.is_diff: log.debug(f"Writing {len(weather_data)} records to pickle file") weather_data.to_pickle(pickle_file) return weather_data
async def _fetch_covid19_data() -> pd.DataFrame: log.debug("Fetching data from github covid dataset") return pd.read_csv( "https://raw.githubusercontent.com/datasets/covid-19/master/data/time-series-19-covid-combined.csv")
def __init__(self, cache_path=Path(__file__).parent): self.api_key = environ.get("DARKSKY_KEY", None) log.debug(f"Darksky api key: {self.api_key}") self.cache_path = cache_path self.is_diff = True
getmtime(pickle_file)) if pickle_file.exists() else ctime(time()) if hasattr(csv, 'decode'): csv = csv.decode('utf8') return (TEMPLATE % dict(csv=csv, modified_time=modified_time, kwargs=json.dumps(kwargs))).encode() @app.route('/') def home(): loop = asyncio.new_event_loop() html = loop.run_until_complete( pivot_ui(path=app.instance_path, **kwargs)) return html.decode() return app if __name__ == '__main__': instance_path = Path(__file__).parent if (instance_path / '.env').exists(): logger.debug(f"Loading {instance_path / '.env'}") load_dotenv(instance_path / '.env') app = covid19_webapp(instance_path=instance_path, rows=["Country/Region"], cols=["Date"], aggregatorName="Maximum", vals=["Deaths"], rendererName="Line Chart") app.run(host="0.0.0.0", port=8000, load_dotenv=True)