def test_transform_static_features_without_population(self): preprocessing_config = FeaturePreprocessingConfig( imputation_strategy='median', standardize=True) model = base.BaseCovidModelDefinition( static_preprocessing_config=preprocessing_config) raw_data = { constants.HOUSEHOLDS: { '01': 2, '02': None, '03': 40, '04': None, }, constants.HOSPITAL_RATING_AVERAGE: { '01': 2, '02': 5, '03': None, '04': 3, }, } model.get_static_features = mock.Mock( return_value={ constants.HOUSEHOLDS: constants.HOUSEHOLDS, constants.HOSPITAL_RATING_AVERAGE: constants.HOSPITAL_RATING_AVERAGE, }) with self.assertRaisesRegex(ValueError, 'Static features must include population'): model.transform_static_features(raw_data)
def test_transform_ts_features_without_confirmed_feature(self): preprocessing_config = FeaturePreprocessingConfig( imputation_strategy='median', standardize=True) model = base.BaseCovidModelDefinition( static_preprocessing_config=preprocessing_config) # Missing confirmed feature raw_data = { constants.DEATH: { '01': np.array([0, 1, 2, np.nan, 4]), '02': np.array([1, 2, 3, 4, np.nan]), '03': np.array([np.nan, 3, 4, 5, 6]), }, } static_raw_data = { constants.POPULATION: { '01': 10, '02': 20, '03': 30, }, } model.get_static_features = mock.Mock(return_value={ constants.POPULATION: constants.POPULATION, }) static_transformed_data, _ = model.transform_static_features( static_raw_data) model.get_ts_features = mock.Mock( return_value={ constants.DEATH: constants.DEATH, constants.MOBILITY_INDEX: constants.MOBILITY_INDEX, }) with self.assertRaisesRegex(ValueError, 'confirmed must be in the input features'): model.transform_ts_features(raw_data, static_transformed_data, 5)
def test_preprocess_static_data(self): preprocessing_config = FeaturePreprocessingConfig( imputation_strategy='median', standardize=True) model = base.BaseCovidModelDefinition( static_preprocessing_config=preprocessing_config) raw_data = { constants.POPULATION: { '01': 10, '02': 20, '03': 30, '04': 40, }, constants.HOUSEHOLDS: { '01': 2, '02': None, '03': 40, '04': None, }, constants.HOSPITAL_RATING_AVERAGE: { '01': 2, '02': 5, '03': None, '04': 3, }, } model.get_static_features = mock.Mock( return_value={ constants.POPULATION: constants.POPULATION, constants.HOUSEHOLDS: constants.HOUSEHOLDS, constants.HOSPITAL_RATING_AVERAGE: constants.HOSPITAL_RATING_AVERAGE, }) # Normalizes everything but Population by default expected_data = { constants.POPULATION: { '01': 10, '02': 20, '03': 30, '04': 40 }, constants.HOUSEHOLDS: { '01': 0, '02': 19 / 38, '03': 1, '04': 19 / 39 }, constants.HOSPITAL_RATING_AVERAGE: { '01': 0, '02': 1, '03': 1 / 3, '04': 1 / 3 }, } transformed_data, _ = model.transform_static_features(raw_data) # Need to get around floating point issues so using pandas series for feature in expected_data: self.assertIn(feature, transformed_data) expected_feature = pd.Series(expected_data[feature]) transformed_data_feature = pd.Series(expected_data[feature]) pd.testing.assert_series_equal(expected_feature, transformed_data_feature)
def test_preprocess_ts_data(self): preprocessing_config = FeaturePreprocessingConfig( imputation_strategy='median', standardize=True, ffill_features=True, bfill_features=True) model = base.BaseCovidModelDefinition( static_preprocessing_config=preprocessing_config) # Needs deaths and confirmed raw_data = { constants.DEATH: { '01': np.array([0, 1, 2, np.nan, 4]), '02': np.array([1, 2, 3, 4, np.nan]), '03': np.array([np.nan, 3, 4, 5, 6]), }, constants.CONFIRMED: { '01': np.array([10, 11, 12, np.nan, 14]), '02': np.array([11, 21, 31, 41, np.nan]), '03': np.array([np.nan, 13, 14, 15, 16]), }, constants.MOBILITY_INDEX: { '01': np.array([4, 3, 2, np.nan, 4]), '02': np.array([np.nan, 1, 2, 1, 4]), '03': np.array([10, 1, 1, np.nan, 1]), }, } expected_data = { constants.DEATH: { '01': np.array([0, 1, 2, np.nan, 4]), '02': np.array([1, 2, 3, 4, np.nan]), '03': np.array([np.nan, 3, 4, 5, 6]), }, constants.CONFIRMED: { '01': np.array([10, 11, 12, np.nan, 14]), '02': np.array([11, 21, 31, 41, np.nan]), '03': np.array([np.nan, 13, 14, 15, 16]), }, constants.MOBILITY_INDEX: { '01': (np.array([4, 3, 2, 2, 4]) - 1.0) / (10 - 1.0), '02': (np.array([1, 1, 2, 1, 4]) - 1.0) / (10 - 1.0), '03': (np.array([10, 1, 1, 1, 1]) - 1.0) / (10.0 - 1), }, } static_raw_data = { constants.POPULATION: { '01': 10, '02': 20, '03': 30, }, constants.HOUSEHOLDS: { '01': 2, '02': None, '03': 40, }, constants.HOSPITAL_RATING_AVERAGE: { '01': 2, '02': 5, '03': None, }, } model.get_static_features = mock.Mock( return_value={ constants.POPULATION: constants.POPULATION, constants.HOUSEHOLDS: constants.HOUSEHOLDS, constants.HOSPITAL_RATING_AVERAGE: constants.HOSPITAL_RATING_AVERAGE, }) static_transformed_data, _ = model.transform_static_features( static_raw_data) model.get_ts_features = mock.Mock( return_value={ constants.DEATH: constants.DEATH, constants.MOBILITY_INDEX: constants.MOBILITY_INDEX, }) # Normalizes nothing by default but adds 4 features. added_features = { constants.DEATH_PREPROCESSED, constants.DEATH_PREPROCESSED_MEAN_TO_SUM_RATIO, constants.CONFIRMED_PREPROCESSED, constants.CONFIRMED_PREPROCESSED_MEAN_TO_SUM_RATIO } transformed_data, _ = model.transform_ts_features( raw_data, static_transformed_data, 5) for new_feature in added_features: self.assertIn(new_feature, transformed_data) del transformed_data[new_feature] for feature in raw_data: for loc in raw_data[feature]: np.testing.assert_array_almost_equal( transformed_data[feature][loc], raw_data[feature][loc]) # Add mobility to be pre-processed model.get_ts_features_to_preprocess = mock.Mock( return_value={constants.MOBILITY_INDEX}) # Will now pre-process mobility index transformed_data, _ = model.transform_ts_features( raw_data, static_raw_data, 5) for new_feature in added_features: self.assertIn(new_feature, transformed_data) for feature in expected_data: for loc in expected_data[feature]: np.testing.assert_array_almost_equal( transformed_data[feature][loc], expected_data[feature][loc])