async def test_client(aiohttp_client): app = web.Application() config.get_config(app, DEFAULT_CONFIG_PATH) app.add_routes(routes.routes) app.redis = redis.Redis(host='redis_test') app.on_startup.extend([init_db.init_database]) app.on_cleanup.extend([init_db.close_database]) return await aiohttp_client(app)
def init_app(argv=None) -> web.Application: args = parse_args() app = web.Application() app.add_routes(routes.routes) config.get_config(app, args.config_path) app.redis = redis.Redis(host='redis') app.on_startup.extend([init_db.init_database]) app.on_cleanup.extend([init_db.close_database]) return app
def run(self): city_stats_location = 'Phase 2 - Districts' config = get_config() repository = AWSFileRepository(config['aws']['bucket production']) if repository.exists(city_stats_location): city_stats = repository.get_dataframe(city_stats_location) # -- sort by date city_stats = city_stats.sort_values(by=['date']) # critical cities critical_city_stats = critical_districts(data=city_stats) # download to local fs critical_city_stats.to_csv(self.output().path, index=False) else: log.error("Missing City Stats Data") # test new_data = critical_districts(data=city_stats) print("critical cities") print(new_data) print(new_data.columns) return None
def run(self): # TODO: it needs to partialy fail if one of its dependencies fails # as in it should continue processing all the other wards even if a few failed. ward_storage_location = 'raw_ward_data' config = get_config() repository = AWSFileRepository(config['aws']['bucket production']) if repository.exists(ward_storage_location): all_wards = repository.get_dataframe(ward_storage_location) all_wards = all_wards.set_index( ['state', 'district', 'ward', 'date']) else: all_wards = None for district, ward_task in self.input().items(): log.info(f'Processing: {district}') with ward_task.open('r') as json_file: ward_task = pd.read_csv(json_file, parse_dates=['date']) ward_task = ward_task.set_index( ['state', 'district', 'ward', 'date']) # This needs to support overwriting existing data as well as adding new data # TODO make a test for it if all_wards is None: all_wards = ward_task else: all_wards = all_wards.combine_first( ward_task) # update old values and add new values # cleanup for task in self.input().values(): task.remove() # store the raw data, no imputation done yet repository.store_dataframe(all_wards, ward_storage_location, allow_create=True, store_index=True) # impute delta's atleast for Mumbai this is needed it only provides totals delta_needed_for = [ 'tested', 'confirmed', 'recovered', 'deceased', 'active', 'other' ] group_by_cols = ['state', 'district', 'ward'] all_wards = interpolate_values(all_wards, group_by_cols, delta_needed_for) all_wards = create_delta_cols(all_wards, group_by_cols, delta_needed_for) # add population static_ward_data = get_static_ward_data() all_wards = all_wards.join(static_ward_data, on=group_by_cols, how='left') all_wards.to_csv(self.output().path, index=True)
def setUp(self) -> None: config = get_config() self._url = config['google sheets']['url development'] self._sheets_created = [] self._test_storage_location = 'for_integration_tests' # be sure there is a basic worksheet to run other tests on. # but we are not deleting this one. We just want to make sure its always there self.test_create_storage_location()
def run(self): ward_stats_location = 'Phase 2 - Wards' config = get_config() repository = AWSFileRepository(config['aws']['bucket production']) if repository.exists(ward_stats_location): ward_stats = repository.get_dataframe(ward_stats_location) # -- sort by date ward_stats = ward_stats.sort_values(by = ['date']) # download to local fs ward_stats.to_csv(self.output().path, index=False) else: log.error("Missing Ward Stats Data") return None
def run(self): if 'stopcoronavirus' in self.file_url: dashboard_storage = 'mumbai_dashboard.pdf' config = get_config() repository = AWSFileRepository(config['aws']['bucket production']) if repository.exists(dashboard_storage): repository._download_file(self._temp_file.path, config['aws']['bucket production'], dashboard_storage) else: print("Mumbai file not found in S3 bucket") else: response = requests.get( self.file_url, verify='tasks/districts/mumbaiwards_consolidated.pem') with open(self.output().path, mode='wb') as output_file: output_file.write(response.content)
def run(self): config = get_config() repository = AWSFileRepository(config['aws']['bucket production']) # import master districts data all_districts = repository.get_dataframe(self.s3_districts_path) # delete existing RT/DT columns avl_cols = [ x for x in all_districts.columns if x in ['mean.RT', 'upper.RT', 'lower.RT', 'dt'] ] for avl_col in avl_cols: del all_districts[avl_col] # read RT results rt_results0 = pd.read_csv(self.local_rt_path, parse_dates=["date"]) # specify column name containing Rt values rt_colname = ['mean', 'upper', 'lower'] rt_results = rt_results0[['city', 'date'] + rt_colname] rt_results.columns = [ 'district', 'date', 'mean.RT', 'upper.RT', 'lower.RT' ] # join rt data with all districts data all_districts = all_districts.merge(rt_results, on=['district', 'date'], how='left') # read DT results dt_results0 = pd.read_csv(self.local_dt_path, parse_dates=["date"]) # pick only the relevant columns dt_results = dt_results0[["district", "date", "dt"]] all_districts = all_districts.merge(dt_results, on=['district', 'date'], how='left') # push RT/DT Critical Cities Updates to Repo repository.store_dataframe(all_districts, self.s3_districts_update_path, allow_create=True)
def create_app(test_config=None): """ Flask application factory Load applicaiton configuration, creates Flask instance and attach some hook to it The method is called using 'flask' command line with FLASK_APP=backend as environmnt variable. """ app = Flask(__name__) app.config.from_mapping( mongodb={}, ) if test_config is None: app.config.update(get_config()) else: app.config.update(test_config) @app.route('/') def index(): return "Welcome Paranuara Citizen API" @app.errorhandler(BusinessException) def handle_not_found(error): response = jsonify(error.to_dict()) response.status_code = error.status_code return response # Register application commands db.init_app(app) doc.init_app(app) app.register_blueprint(people_v1, url_prefix='/v1/people') app.register_blueprint(companies_v1, url_prefix='/v1/companies') app.after_request(add_cors_headers) return app
def run(self): config = get_config() # we are skipping older data since we only have low case numbers there. start_date = datetime.strptime(config['dashboard']['start date'], '%Y-%m-%d') # repository = GSheetRepository(config['google sheets']['url production']) repository = AWSFileRepository(config['aws']['bucket production']) fetch_covid19_india_task = self.input()['state_district_data'] fetch_wards_task = self.input()['ward_data'] with fetch_covid19_india_task.open('r') as json_file: all_covid19india_data = json.load(json_file) all_ward_data = pd.read_csv(fetch_wards_task.path, parse_dates=['date']) # cleanup fetch_covid19_india_task.remove() fetch_wards_task.remove() state_data, district_data = ExtractCovid19IndiaData().process( all_covid19india_data) # not the best location to create this, but it's ok for now if not repository.exists(self.storage_hospitalizations): df = pd.DataFrame({'date': [], 'percentages': []}) repository.store_dataframe(df, self.storage_hospitalizations, allow_create=True) hospitalization_df = repository.get_dataframe( self.storage_hospitalizations) hospitalizations_updated = impute_hospitalization_percentages( hospitalization_df, state_data['date']) state_data = state_data[state_data['date'] >= start_date] state_data = extend_and_impute_metrics( raw_metrics=state_data, hospitalizations=hospitalizations_updated, grouping_columns=['state']) district_data = district_data[district_data['date'] >= start_date] district_data = extend_and_impute_metrics( raw_metrics=district_data, hospitalizations=hospitalizations_updated, grouping_columns=['state', 'district']) ward_data = all_ward_data[all_ward_data['date'] >= start_date] ward_data = extend_and_impute_metrics( raw_metrics=ward_data, hospitalizations=hospitalizations_updated, grouping_columns=['state', 'district', 'ward']) self.update_population_sheets(state_data, district_data, repository) # Idea placeholder # Calculate today's top 20ish cities and add that top 20 as a tab in the google sheet so the dashboard can # get access to it. # validate and filter self.states_is_valid = self._has_all_columns( state_data, self.state_columns_needed_by_dashboard) self.districts_is_valid = self._has_all_columns( district_data, self.district_columns_needed_by_dashboard) self.wards_is_valid = self._has_all_columns( ward_data, self.ward_columns_needed_by_dashboard) states_filtered = state_data[self.state_columns_needed_by_dashboard] districts_filtered = district_data[ self.district_columns_needed_by_dashboard] wards_filtered = ward_data[self.ward_columns_needed_by_dashboard] repository.store_dataframe(hospitalizations_updated, self.storage_hospitalizations, allow_create=True) repository.store_dataframe(states_filtered, self.storage_states, allow_create=True) repository.store_dataframe(districts_filtered, self.storage_districts, allow_create=True) repository.store_dataframe(wards_filtered, self.storage_wards, allow_create=True)
def run(self): config = get_config() # we are skipping older data since we only have low case numbers there. start_date = datetime.strptime(config['dashboard']['start date'], '%Y-%m-%d') # repository = GSheetRepository(config['google sheets']['url production']) repository = AWSFileRepository(config['aws']['bucket production']) #fetch_covid19_india_task = self.input()['state_district_data'] fetch_wards_task = self.input()['ward_data'] fetch_districtoverview_task = self.input()['district_overview_data'] #with fetch_covid19_india_task.open('r') as json_file: # all_covid19india_data = json.load(json_file) states_covid19india_data = pd.read_csv( "https://api.covid19tracker.in/data/csv/latest/states.csv", parse_dates=["Date"]) states_covid19india_data['other'] = None states_covid19india_data['tested'] = None districts_covid19india_data = pd.read_csv( "https://api.covid19tracker.in/data/csv/latest/districts.csv", parse_dates=["Date"]) districts_covid19india_data['tested'] = None # read population data states_pop = pd.read_csv("backend/data/static_states_data.csv")[[ 'state', 'population' ]] districts_pop = pd.read_csv("backend/data/static_districts_data.csv")[[ 'state', 'district', 'population' ]] # obtain latest time series at state/district levels states_covid19india_data.columns = [ 'date', 'state', 'total.confirmed', 'total.recovered', 'total.deceased', 'total.other', 'total.tested' ] states_covid19india_data['tested'] = states_covid19india_data[ 'total.tested'] states_covid19india_data = states_covid19india_data.merge(states_pop, on=["state"], how="left") districts_covid19india_data.columns = [ 'date', 'state', 'district', 'total.confirmed', 'total.recovered', 'total.deceased', 'total.other', 'total.tested' ] districts_covid19india_data['tested'] = districts_covid19india_data[ 'total.tested'] districts_covid19india_data = districts_covid19india_data.merge( districts_pop, on=["state", "district"], how="left") all_ward_data = pd.read_csv(fetch_wards_task.path, parse_dates=['date']) district_overview_data = pd.read_csv(fetch_districtoverview_task.path, parse_dates=['date']) # cleanup #fetch_covid19_india_task.remove() fetch_wards_task.remove() state_data, district_data = ExtractCovid19IndiaData().process( states_covid19india_data, districts_covid19india_data) # not the best location to create this, but it's ok for now if not repository.exists(self.storage_hospitalizations): df = pd.DataFrame({'date': [], 'percentages': []}) repository.store_dataframe(df, self.storage_hospitalizations, allow_create=True) hospitalization_df = repository.get_dataframe( self.storage_hospitalizations) hospitalizations_updated = impute_hospitalization_percentages( hospitalization_df, state_data['date']) state_data = state_data[state_data['date'] >= start_date] state_data = extend_and_impute_metrics( raw_metrics=state_data, hospitalizations=hospitalizations_updated, grouping_columns=['state']) district_data = district_data[district_data['date'] >= start_date] district_data = extend_and_impute_metrics( raw_metrics=district_data, hospitalizations=hospitalizations_updated, grouping_columns=['state', 'district']) ward_data = all_ward_data[all_ward_data['date'] >= start_date] ward_data = extend_and_impute_metrics( raw_metrics=ward_data, hospitalizations=hospitalizations_updated, grouping_columns=['state', 'district', 'ward']) #district_overview_data = district_overview_data.set_index(['date']) self.update_population_sheets(state_data, district_data, repository) # Idea placeholder # Calculate today's top 20ish cities and add that top 20 as a tab in the google sheet so the dashboard can # get access to it. # validate and filter self.states_is_valid = self._has_all_columns( state_data, self.state_columns_needed_by_dashboard) self.districts_is_valid = self._has_all_columns( district_data, self.district_columns_needed_by_dashboard) self.wards_is_valid = self._has_all_columns( ward_data, self.ward_columns_needed_by_dashboard) self.districtoverview_is_valid = self._has_all_columns( district_overview_data, self.districtoverview_columns_needed_by_dashboard) states_filtered = state_data[self.state_columns_needed_by_dashboard] districts_filtered = district_data[ self.district_columns_needed_by_dashboard] wards_filtered = ward_data[self.ward_columns_needed_by_dashboard] districtoverview_filtered = district_overview_data[ ['date'] + self.districtoverview_columns_needed_by_dashboard] repository.store_dataframe(hospitalizations_updated, self.storage_hospitalizations, allow_create=True) repository.store_dataframe(states_filtered, self.storage_states, allow_create=True) repository.store_dataframe(districts_filtered, self.storage_districts, allow_create=True) repository.store_dataframe(wards_filtered, self.storage_wards, allow_create=True) repository.store_dataframe(districtoverview_filtered, self.storage_districtoverview, allow_create=True)
def test_config(self): result = get_config() self.assertIsNotNone(result)
def test_config_gsheet(self): result = get_config() self.assertTrue('google sheets' in result) self.assertTrue('url production' in result['google sheets']) self.assertTrue('url development' in result['google sheets'])
def run(self): # TODO: it needs to partialy fail if one of its dependencies fails # as in it should continue processing all the other wards even if a few failed. overall_storage_location = 'raw_district_overview_stats' config = get_config() repository = AWSFileRepository(config['aws']['bucket production']) if repository.exists(overall_storage_location): overall_df = repository.get_dataframe(overall_storage_location) overall_df = overall_df.set_index(['date', 'metric', 'metric_type']) else: overall_df = None for district, overall_task in self.input().items(): log.info(f'Processing: {district}') with overall_task.open('r') as json_file: overall_task = pd.read_csv(json_file, parse_dates=['date']) overall_task = overall_task.set_index(['date', 'metric', 'metric_type']) # This needs to support overwriting existing data as well as adding new data # TODO make a test for it if overall_df is None: overall_df = overall_task else: overall_df = overall_df.combine_first(overall_task) # update old values and add new values # cleanup for task in self.input().values(): task.remove() # store the raw data, no imputation done yet repository.store_dataframe(overall_df, overall_storage_location, allow_create=True, store_index=True) data_mini = overall_df.reset_index(drop=False) data_mini = data_mini.loc[data_mini['metric'].isin(['active.ccc1.facilities','active.ccc2.facilities','contact.traced.high.risk','contact.traced.low.risk', 'total.contact.traced','containment.zones.active.slums.chawls', 'containment.zones.active.micro.sealed.buildings','floors.sealed','bed.available.dchc.dch.ccc2', 'bed.occupied.dchc.dch.ccc2','bed.available.dchc.dch','bed.occupied.dchc.dch','bed.available.icu', 'bed.occupied.icu','bed.available.o2','bed.occupied.o2','bed.available.ventilator', 'bed.occupied.ventilator','active.critical','total.deaths','total.discharged', 'active.symptomatic','active.asymptomatic','total.tests','total.active','total.positive', 'currently.quarantined.home'])] data_mini['value'] = data_mini.apply(lambda x: x['past.24hrs'] if x['metric'] in ['contact.traced.high.risk', 'contact.traced.low.risk', 'total.contact.traced'] else (x['num.facilities'] if x['metric'] in ['active.ccc1.facilities', 'active.ccc2.facilities'] else x['count']), axis=1) data_mini2 = data_mini.pivot(index='date', columns='metric', values='value') # impute delta's atleast for Mumbai this is needed it only provides totals delta_needed_for = ['deaths', 'discharged', 'tests', 'positive'] group_by_cols = [] try: data_mini2 = interpolate_values_generic(data_mini2, group_by_cols, list(data_mini2)) data_mini2 = create_delta_cols(data_mini2, group_by_cols, delta_needed_for) except: print("error in delta/interpolate") pass # add population data_mini2.to_csv(self.output().path, index=True)