def _standardize_data(cls, data: pd.DataFrame) -> pd.DataFrame: data = dataset_utils.strip_whitespace(data) # Drop all data outside the USA to speed up debugging data = data.loc[data[cls.Fields.COUNTRY].isin(["US", "USA"])] # TODO Figure out how to rename to some ISO standard. country_remap = { "Mainland China": "China", "Bahamas, The": "Bahamas", "Congo (Brazzaville)": "Congo", "Congo (Kinshasa)": "Congo", "Diamond Princess": "Cruise Ship", "Hong Kong SAR": "Hong Kong", "Iran (Islamic Republic of)": "Iran", "Korea, South": "South Korea", "Taiwan*": "Taiwan", "UK": "United Kingdom", "US": "USA", } data = data.replace({cls.Fields.COUNTRY: country_remap}) states = data[cls.Fields.STATE].apply(dataset_utils.parse_state) county_from_state = data[cls.Fields.STATE].apply( dataset_utils.parse_county_from_state) data[cls.Fields.COUNTY] = data[cls.Fields.COUNTY].combine_first( county_from_state) data[cls.Fields.STATE] = states data = cls._fill_incomplete_county_data(data) state_only = data[cls.Fields.FIPS].isnull() & data[ cls.Fields.COUNTY].isnull() # Pad fips values to 5 spots data[cls.Fields.FIPS] = data[cls.Fields.FIPS].apply( lambda x: f"{x.zfill(5)}" if type(x) == str else x) data[cls.Fields.AGGREGATE_LEVEL] = numpy.where(state_only, "state", "county") data = cls._aggregate_fips_data(data) dataset_utils.assert_counties_have_fips(data, county_key=cls.Fields.COUNTY, fips_key=cls.Fields.FIPS) # Not including cases grouped in recovered state. # on 8/11 recovered cases were assigned to a specific fips which caused duplicates. is_recovered_state = data[cls.Fields.STATE] == "Recovered" data.loc[is_recovered_state, cls.Fields.FIPS] = None common_fields_data = cls._rename_to_common_fields(data) return common_fields_data
def standardize_data(cls, data: pd.DataFrame) -> pd.DataFrame: data = dataset_utils.strip_whitespace(data) # TODO Figure out how to rename to some ISO standard. country_remap = { "Mainland China": "China", "Bahamas, The": "Bahamas", "Congo (Brazzaville)": "Congo", "Congo (Kinshasa)": "Congo", "Diamond Princess": "Cruise Ship", "Hong Kong SAR": "Hong Kong", "Iran (Islamic Republic of)": "Iran", "Korea, South": "South Korea", "Taiwan*": "Taiwan", "UK": "United Kingdom", "US": "USA", } data = data.replace({cls.Fields.COUNTRY: country_remap}) states = data[cls.Fields.STATE].apply(dataset_utils.parse_state) county_from_state = data[cls.Fields.STATE].apply( dataset_utils.parse_county_from_state) data[cls.Fields.COUNTY] = data[cls.Fields.COUNTY].combine_first( county_from_state) data[cls.Fields.STATE] = states data = cls._fill_incomplete_county_data(data) state_only = data[cls.Fields.FIPS].isnull() & data[ cls.Fields.COUNTY].isnull() # Pad fips values to 5 spots data[cls.Fields.FIPS] = data[cls.Fields.FIPS].apply( lambda x: f"{x.zfill(5)}" if type(x) == str else x) data[cls.Fields.AGGREGATE_LEVEL] = numpy.where(state_only, "state", "county") data = cls._aggregate_fips_data(data) dataset_utils.assert_counties_have_fips(data, county_key=cls.Fields.COUNTY, fips_key=cls.Fields.FIPS) return data