def test_nyc_aggregation(are_boroughs_zero): nyc_county_fips = custom_aggregations.NEW_YORK_COUNTY_FIPS nyc_borough_fips = custom_aggregations.NYC_BOROUGH_FIPS[0] nyc_cases = 10 borough_cases = 0 if are_boroughs_zero else 10 rows = [ default_timeseries_row(fips=nyc_county_fips, cases=nyc_cases), default_timeseries_row(fips=nyc_borough_fips, cases=borough_cases, deaths=borough_cases, recovered=borough_cases), default_timeseries_row() ] df = pd.DataFrame(rows) # Todo: figure out a better way to define these groups. group = [ 'date', 'source', 'country', 'aggregate_level', 'state', 'generated' ] result = custom_aggregations.update_with_combined_new_york_counties( df, group, are_boroughs_zero=are_boroughs_zero) results = result.sort_values('fips').to_dict(orient='records') assert len(results) == 2 nyc_result = results[1] if are_boroughs_zero: assert nyc_result['cases'] == nyc_cases else: assert nyc_result['cases'] == nyc_cases + borough_cases
def test_nyc_aggregation(are_boroughs_zero): nyc_county_fips = custom_aggregations.NEW_YORK_COUNTY_FIPS nyc_borough_fips = custom_aggregations.NYC_BOROUGH_FIPS[0] nyc_cases = 10 borough_cases = 0 if are_boroughs_zero else 10 rows = [ default_timeseries_row(fips=nyc_county_fips, cases=nyc_cases), default_timeseries_row( fips=nyc_borough_fips, cases=borough_cases, deaths=borough_cases, recovered=borough_cases, ), default_timeseries_row(), ] df = pd.DataFrame(rows) # Todo: figure out a better way to define these groups. group = ["date", "source", "country", "aggregate_level", "state"] result = custom_aggregations.update_with_combined_new_york_counties( df, group, are_boroughs_zero=are_boroughs_zero) results = result.sort_values("fips").to_dict(orient="records") assert len(results) == 2 nyc_result = results[1] if are_boroughs_zero: assert nyc_result["cases"] == nyc_cases else: assert nyc_result["cases"] == nyc_cases + borough_cases assert pd.isna(nyc_result["current_icu"])
def from_source(cls, source: "DataSource", fill_missing_state: bool = True) -> "TimeseriesDataset": """Loads data from a specific datasource. Args: source: DataSource to standardize for timeseries dataset fill_missing_state: If True, backfills missing state data by calculating county level aggregates. Returns: Timeseries object. """ data = source.data group = [ CommonFields.DATE, CommonFields.COUNTRY, CommonFields.AGGREGATE_LEVEL, CommonFields.STATE, ] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=source.HAS_AGGREGATED_NYC_BOROUGH) if fill_missing_state: state_groupby_fields = [ CommonFields.DATE, CommonFields.COUNTRY, CommonFields.STATE, ] non_matching = dataset_utils.aggregate_and_get_nonmatching( data, state_groupby_fields, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) is_state = data[ CommonFields.AGGREGATE_LEVEL] == AggregationLevel.STATE.value state_fips = data.loc[is_state, CommonFields.STATE].map( us_state_abbrev.ABBREV_US_FIPS) data.loc[is_state, CommonFields.FIPS] = state_fips no_fips = data[CommonFields.FIPS].isnull() if no_fips.any(): _log.warning("Dropping rows without FIPS", source=str(source), rows=repr(data.loc[no_fips])) data = data.loc[~no_fips] dups = data.duplicated(COMMON_FIELDS_TIMESERIES_KEYS, keep=False) if dups.any(): raise DuplicateDataException(f"Duplicates in {source}", data.loc[dups]) # Choosing to sort by date data = data.sort_values(CommonFields.DATE) return cls(data, provenance=source.provenance)
def from_source(cls, source: "DataSource", fill_missing_state: bool = True) -> "TimeseriesDataset": """Loads data from a specific datasource. Args: source: DataSource to standardize for timeseries dataset fill_missing_state: If True, backfills missing state data by calculating county level aggregates. Returns: Timeseries object. """ data = source.data # TODO(tom): Do this renaming upstream, when the source is loaded or when first copied from the third party. to_common_fields = { value: key for key, value in source.all_fields_map().items() } final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] group = [ CommonFields.DATE, CommonFields.COUNTRY, CommonFields.AGGREGATE_LEVEL, CommonFields.STATE, ] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=source.HAS_AGGREGATED_NYC_BOROUGH) if fill_missing_state: state_groupby_fields = [ CommonFields.DATE, CommonFields.COUNTRY, CommonFields.STATE, ] non_matching = dataset_utils.aggregate_and_get_nonmatching( data, state_groupby_fields, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) is_state = data[ CommonFields.AGGREGATE_LEVEL] == AggregationLevel.STATE.value state_fips = data.loc[is_state, CommonFields.STATE].map( us_state_abbrev.ABBREV_US_FIPS) data.loc[is_state, CommonFields.FIPS] = state_fips # Choosing to sort by date data = data.sort_values(CommonFields.DATE) return cls(data)
def from_source(cls, source: "DataSource", fill_missing_state=True): """Loads data from a specific datasource. Remaps columns from source dataset, fills in missing data by computing aggregates, and adds standardized county names from fips. Args: source: Data source. fill_missing_state: If True, fills in missing state level data by aggregating county level for a given state. """ if not source.BEDS_FIELD_MAP: raise ValueError("Source must have beds field map.") data = source.data to_common_fields = { value: key for key, value in source.BEDS_FIELD_MAP.items() } final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] data[cls.Fields.SOURCE] = source.SOURCE_NAME data[cls.Fields.GENERATED] = False # Generating max bed count. columns_to_consider = [ cls.Fields.STAFFED_BEDS, cls.Fields.LICENSED_BEDS ] data[cls.Fields.MAX_BED_COUNT] = data[columns_to_consider].max(axis=1) # When grouping nyc data, we don't want to count the generated field # as a value to sum. group = cls.STATE_GROUP_KEY + [cls.Fields.GENERATED] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=False) if fill_missing_state: non_matching = dataset_utils.aggregate_and_get_nonmatching( data, cls.STATE_GROUP_KEY, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() non_matching[cls.Fields.GENERATED] = True data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) return cls(data)
def from_source(cls, source: "DataSource", fill_missing_state=True): """Loads data from a specific datasource.""" if not source.TIMESERIES_FIELD_MAP: raise ValueError("Source must have field timeseries field map.") data = source.data to_common_fields = { value: key for key, value in source.TIMESERIES_FIELD_MAP.items() } final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] data[cls.Fields.SOURCE] = source.SOURCE_NAME data[cls.Fields.GENERATED] = False group = [ cls.Fields.DATE, cls.Fields.SOURCE, cls.Fields.COUNTRY, cls.Fields.AGGREGATE_LEVEL, cls.Fields.STATE, cls.Fields.GENERATED, ] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=True ) if fill_missing_state: state_groupby_fields = [ cls.Fields.DATE, cls.Fields.SOURCE, cls.Fields.COUNTRY, cls.Fields.STATE, ] non_matching = dataset_utils.aggregate_and_get_nonmatching( data, state_groupby_fields, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() non_matching[cls.Fields.GENERATED] = True data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) # Choosing to sort by date data = data.sort_values(cls.Fields.DATE) return cls(data)
def from_source(cls, source: "DataSource", fill_missing_state=True): """Loads data from a specific datasource. Remaps columns from source dataset, fills in missing data by computing aggregates, and adds standardized county names from fips. """ if not source.BEDS_FIELD_MAP: raise ValueError("Source must have beds field map.") data = source.data to_common_fields = { value: key for key, value in source.BEDS_FIELD_MAP.items() } final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] data[cls.Fields.SOURCE] = source.SOURCE_NAME data[cls.Fields.GENERATED] = False group = [ cls.Fields.SOURCE, cls.Fields.AGGREGATE_LEVEL, cls.Fields.STATE, cls.Fields.GENERATED ] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=False) if fill_missing_state: state_groupby_fields = [cls.Fields.SOURCE, cls.Fields.STATE] non_matching = dataset_utils.aggregate_and_get_nonmatching( data, state_groupby_fields, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() non_matching[cls.Fields.GENERATED] = True data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) return cls(data)
def _aggregate_new_york_data(cls, data): # When grouping nyc data, we don't want to count the generated field # as a value to sum. nyc_data = data[data[cls.Fields.FIPS].isin( custom_aggregations.ALL_NYC_FIPS)] if not len(nyc_data): return data group = cls.STATE_GROUP_KEY weighted_all_bed_occupancy = None if cls.Fields.ALL_BED_TYPICAL_OCCUPANCY_RATE in data.columns: licensed_beds = nyc_data[cls.Fields.LICENSED_BEDS] occupancy_rates = nyc_data[ cls.Fields.ALL_BED_TYPICAL_OCCUPANCY_RATE] weighted_all_bed_occupancy = ( (licensed_beds * occupancy_rates).sum() / licensed_beds.sum()) weighted_icu_occupancy = None if cls.Fields.ICU_TYPICAL_OCCUPANCY_RATE in data.columns: icu_beds = nyc_data[cls.Fields.ICU_BEDS] occupancy_rates = nyc_data[cls.Fields.ICU_TYPICAL_OCCUPANCY_RATE] weighted_icu_occupancy = ((icu_beds * occupancy_rates).sum() / icu_beds.sum()) data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=False) nyc_fips = custom_aggregations.NEW_YORK_COUNTY_FIPS if weighted_all_bed_occupancy: data.loc[data[cls.Fields.FIPS] == nyc_fips, cls.Fields.ALL_BED_TYPICAL_OCCUPANCY_RATE] = ( weighted_all_bed_occupancy) if weighted_icu_occupancy: data.loc[data[cls.Fields.FIPS] == nyc_fips, cls.Fields.ICU_TYPICAL_OCCUPANCY_RATE] = ( weighted_icu_occupancy) return data
def from_source(cls, source: "DataSource", fill_missing_state: bool = True, fill_na: bool = True) -> "Timeseries": """Loads data from a specific datasource. Args: source: DataSource to standardize for timeseries dataset fill_missing_state: If True, backfills missing state data by calculating county level aggregates. fill_na: If True, fills in all NaN values for metrics columns. Returns: Timeseries object. """ if not source.TIMESERIES_FIELD_MAP: raise ValueError("Source must have field timeseries field map.") data = source.data to_common_fields = { value: key for key, value in source.TIMESERIES_FIELD_MAP.items() } final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] data[cls.Fields.SOURCE] = source.SOURCE_NAME data[cls.Fields.GENERATED] = False group = [ cls.Fields.DATE, cls.Fields.SOURCE, cls.Fields.COUNTRY, cls.Fields.AGGREGATE_LEVEL, cls.Fields.STATE, cls.Fields.GENERATED, ] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=True) if fill_missing_state: state_groupby_fields = [ cls.Fields.DATE, cls.Fields.SOURCE, cls.Fields.COUNTRY, cls.Fields.STATE, ] non_matching = dataset_utils.aggregate_and_get_nonmatching( data, state_groupby_fields, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() non_matching[cls.Fields.GENERATED] = True data = pd.concat([data, non_matching]) if fill_na: # Filtering out metric columns that don't exist in the dataset. # It might be that we all timeseries datasets to have all of the metric # columns. If so, initialization of the missing columns should come earlier. metric_columns = [ field for field in cls.Fields.metrics() if field in data.columns ] data[metric_columns] = data[metric_columns].fillna(0.0) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) # Choosing to sort by date data = data.sort_values(cls.Fields.DATE) return cls(data)