def standardize_data(cls, data: pd.DataFrame) -> pd.DataFrame: # Add Missing unknown_fips = [] for state in data.state.unique(): row = { cls.Fields.STATE: state, cls.Fields.FIPS: ABBREV_US_UNKNOWN_COUNTY_FIPS[state], cls.Fields.POPULATION: None, cls.Fields.COUNTY: "Unknown", } unknown_fips.append(row) data = data.append(unknown_fips) # All DH data is aggregated at the county level data[cls.Fields.AGGREGATE_LEVEL] = AggregationLevel.COUNTY.value data[cls.Fields.COUNTRY] = "USA" states_aggregated = dataset_utils.aggregate_and_get_nonmatching( data, [cls.Fields.COUNTRY, cls.Fields.STATE, cls.Fields.AGGREGATE_LEVEL], AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() states_aggregated[cls.Fields.FIPS] = states_aggregated[ cls.Fields.STATE].map(ABBREV_US_FIPS) states_aggregated[cls.Fields.COUNTY] = None common_fields_data = pd.concat([data, states_aggregated]) return common_fields_data
def from_source(cls, source: "DataSource", fill_missing_state=True): """Loads data from a specific datasource. Remaps columns from source dataset, fills in missing data by computing aggregates, and adds standardized county names from fips. """ if not source.BEDS_FIELD_MAP: raise ValueError("Source must have beds field map.") data = source.data to_common_fields = { value: key for key, value in source.BEDS_FIELD_MAP.items() } final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] data[cls.Fields.SOURCE] = source.SOURCE_NAME data[cls.Fields.GENERATED] = False if fill_missing_state: state_groupby_fields = [cls.Fields.SOURCE, cls.Fields.STATE] non_matching = dataset_utils.aggregate_and_get_nonmatching( data, state_groupby_fields, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() non_matching[cls.Fields.GENERATED] = True data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) return cls(data)
def standardize_data(cls, data: pd.DataFrame) -> pd.DataFrame: # Add Missing unknown_fips = [] for state in data.state.unique(): row = { cls.Fields.STATE: state, # TODO(chris): Possibly separate fips out by state prefix cls.Fields.FIPS: enums.UNKNOWN_FIPS, cls.Fields.POPULATION: None, cls.Fields.COUNTY: 'Unknown' } unknown_fips.append(row) data = data.append(unknown_fips) # All DH data is aggregated at the county level data[cls.Fields.AGGREGATE_LEVEL] = AggregationLevel.COUNTY.value data[cls.Fields.COUNTRY] = "USA" states_aggregated = dataset_utils.aggregate_and_get_nonmatching( data, [cls.Fields.COUNTRY, cls.Fields.STATE, cls.Fields.AGGREGATE_LEVEL], AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() states_aggregated[cls.Fields.FIPS] = states_aggregated[ cls.Fields.STATE].map(us_state_abbrev.abbrev_us_fips) states_aggregated[cls.Fields.COUNTY] = None return pd.concat([data, states_aggregated])
def from_source(cls, source: "DataSource", fill_missing_state: bool = True) -> "TimeseriesDataset": """Loads data from a specific datasource. Args: source: DataSource to standardize for timeseries dataset fill_missing_state: If True, backfills missing state data by calculating county level aggregates. Returns: Timeseries object. """ data = source.data group = [ CommonFields.DATE, CommonFields.COUNTRY, CommonFields.AGGREGATE_LEVEL, CommonFields.STATE, ] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=source.HAS_AGGREGATED_NYC_BOROUGH) if fill_missing_state: state_groupby_fields = [ CommonFields.DATE, CommonFields.COUNTRY, CommonFields.STATE, ] non_matching = dataset_utils.aggregate_and_get_nonmatching( data, state_groupby_fields, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) is_state = data[ CommonFields.AGGREGATE_LEVEL] == AggregationLevel.STATE.value state_fips = data.loc[is_state, CommonFields.STATE].map( us_state_abbrev.ABBREV_US_FIPS) data.loc[is_state, CommonFields.FIPS] = state_fips no_fips = data[CommonFields.FIPS].isnull() if no_fips.any(): _log.warning("Dropping rows without FIPS", source=str(source), rows=repr(data.loc[no_fips])) data = data.loc[~no_fips] dups = data.duplicated(COMMON_FIELDS_TIMESERIES_KEYS, keep=False) if dups.any(): raise DuplicateDataException(f"Duplicates in {source}", data.loc[dups]) # Choosing to sort by date data = data.sort_values(CommonFields.DATE) return cls(data, provenance=source.provenance)
def from_source(cls, source: "DataSource", fill_missing_state: bool = True) -> "TimeseriesDataset": """Loads data from a specific datasource. Args: source: DataSource to standardize for timeseries dataset fill_missing_state: If True, backfills missing state data by calculating county level aggregates. Returns: Timeseries object. """ data = source.data # TODO(tom): Do this renaming upstream, when the source is loaded or when first copied from the third party. to_common_fields = { value: key for key, value in source.all_fields_map().items() } final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] group = [ CommonFields.DATE, CommonFields.COUNTRY, CommonFields.AGGREGATE_LEVEL, CommonFields.STATE, ] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=source.HAS_AGGREGATED_NYC_BOROUGH) if fill_missing_state: state_groupby_fields = [ CommonFields.DATE, CommonFields.COUNTRY, CommonFields.STATE, ] non_matching = dataset_utils.aggregate_and_get_nonmatching( data, state_groupby_fields, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) is_state = data[ CommonFields.AGGREGATE_LEVEL] == AggregationLevel.STATE.value state_fips = data.loc[is_state, CommonFields.STATE].map( us_state_abbrev.ABBREV_US_FIPS) data.loc[is_state, CommonFields.FIPS] = state_fips # Choosing to sort by date data = data.sort_values(CommonFields.DATE) return cls(data)
def from_source(cls, source: "DataSource", fill_missing_state=True): """Loads data from a specific datasource. Remaps columns from source dataset, fills in missing data by computing aggregates, and adds standardized county names from fips. Args: source: Data source. fill_missing_state: If True, fills in missing state level data by aggregating county level for a given state. """ if not source.BEDS_FIELD_MAP: raise ValueError("Source must have beds field map.") data = source.data to_common_fields = { value: key for key, value in source.BEDS_FIELD_MAP.items() } final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] data[cls.Fields.SOURCE] = source.SOURCE_NAME data[cls.Fields.GENERATED] = False # Generating max bed count. columns_to_consider = [ cls.Fields.STAFFED_BEDS, cls.Fields.LICENSED_BEDS ] data[cls.Fields.MAX_BED_COUNT] = data[columns_to_consider].max(axis=1) # When grouping nyc data, we don't want to count the generated field # as a value to sum. group = cls.STATE_GROUP_KEY + [cls.Fields.GENERATED] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=False) if fill_missing_state: non_matching = dataset_utils.aggregate_and_get_nonmatching( data, cls.STATE_GROUP_KEY, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() non_matching[cls.Fields.GENERATED] = True data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) return cls(data)
def from_source(cls, source: "DataSource", fill_missing_state=True): """Loads data from a specific datasource.""" if not source.TIMESERIES_FIELD_MAP: raise ValueError("Source must have field timeseries field map.") data = source.data to_common_fields = { value: key for key, value in source.TIMESERIES_FIELD_MAP.items() } final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] data[cls.Fields.SOURCE] = source.SOURCE_NAME data[cls.Fields.GENERATED] = False group = [ cls.Fields.DATE, cls.Fields.SOURCE, cls.Fields.COUNTRY, cls.Fields.AGGREGATE_LEVEL, cls.Fields.STATE, cls.Fields.GENERATED, ] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=True ) if fill_missing_state: state_groupby_fields = [ cls.Fields.DATE, cls.Fields.SOURCE, cls.Fields.COUNTRY, cls.Fields.STATE, ] non_matching = dataset_utils.aggregate_and_get_nonmatching( data, state_groupby_fields, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() non_matching[cls.Fields.GENERATED] = True data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) # Choosing to sort by date data = data.sort_values(cls.Fields.DATE) return cls(data)
def from_source(cls, source: "DataSource", fill_missing_state=True): """Loads data from a specific datasource. Remaps columns from source dataset, fills in missing data by computing aggregates, and adds standardized county names from fips. Args: source: Data source. fill_missing_state: If True, fills in missing state level data by aggregating county level for a given state. """ if not source.COMMON_FIELD_MAP and not source.INDEX_FIELD_MAP: raise ValueError("Source must have metadata field map.") data = source.data fields = source.all_fields_map().items() to_common_fields = {value: key for key, value in fields} final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] data = cls._aggregate_new_york_data(data) if fill_missing_state: non_matching = dataset_utils.aggregate_and_get_nonmatching( data, cls.STATE_GROUP_KEY, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() data = pd.concat([data, non_matching]) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) # Add state fips is_state = data[ cls.Fields.AGGREGATE_LEVEL] == AggregationLevel.STATE.value state_fips = data.loc[is_state, cls.Fields.STATE].map( us_state_abbrev.ABBREV_US_FIPS) data.loc[is_state, cls.Fields.FIPS] = state_fips return cls(data)
def from_source(cls, source: "DataSource", fill_missing_state: bool = True, fill_na: bool = True) -> "Timeseries": """Loads data from a specific datasource. Args: source: DataSource to standardize for timeseries dataset fill_missing_state: If True, backfills missing state data by calculating county level aggregates. fill_na: If True, fills in all NaN values for metrics columns. Returns: Timeseries object. """ if not source.TIMESERIES_FIELD_MAP: raise ValueError("Source must have field timeseries field map.") data = source.data to_common_fields = { value: key for key, value in source.TIMESERIES_FIELD_MAP.items() } final_columns = to_common_fields.values() data = data.rename(columns=to_common_fields)[final_columns] data[cls.Fields.SOURCE] = source.SOURCE_NAME data[cls.Fields.GENERATED] = False group = [ cls.Fields.DATE, cls.Fields.SOURCE, cls.Fields.COUNTRY, cls.Fields.AGGREGATE_LEVEL, cls.Fields.STATE, cls.Fields.GENERATED, ] data = custom_aggregations.update_with_combined_new_york_counties( data, group, are_boroughs_zero=True) if fill_missing_state: state_groupby_fields = [ cls.Fields.DATE, cls.Fields.SOURCE, cls.Fields.COUNTRY, cls.Fields.STATE, ] non_matching = dataset_utils.aggregate_and_get_nonmatching( data, state_groupby_fields, AggregationLevel.COUNTY, AggregationLevel.STATE, ).reset_index() non_matching[cls.Fields.GENERATED] = True data = pd.concat([data, non_matching]) if fill_na: # Filtering out metric columns that don't exist in the dataset. # It might be that we all timeseries datasets to have all of the metric # columns. If so, initialization of the missing columns should come earlier. metric_columns = [ field for field in cls.Fields.metrics() if field in data.columns ] data[metric_columns] = data[metric_columns].fillna(0.0) fips_data = dataset_utils.build_fips_data_frame() data = dataset_utils.add_county_using_fips(data, fips_data) # Choosing to sort by date data = data.sort_values(cls.Fields.DATE) return cls(data)