def regression_imputation(df):
'''Returns the dataframe where missing values are imputed using IterativeImputer and BayesianRidge()
This is a regression imputation method.
Parameters:
-----------
df: pd.DataFrame
Returns:
--------
df_result: pd.DataFrame where the missing values are imputed using multiple imputation
'''
df_new = df.copy()
df_new = make_missing_np_nan(df_new)
missing, unique = imputation_heuristic_column(df, 0.99)
df_new = delete_cols(df_new, missing)
df_new = delete_cols(df_new, unique)
cat_cols, date_cols, num_cols = type_cols(df_new)
df_new = df_new[num_cols]
columns = df_new.columns
imputer = IterativeImputer(random_state=0)
imputed = imputer.fit_transform(df_new)
df_imputed = pd.DataFrame(imputed, columns=columns)
not_imputed_cols = cat_cols + date_cols
df_result = pd.concat([df_imputed, df[not_imputed_cols]], axis=1)
return df_result
def MICE_imputation(df, categorical=False, nr_iter=3):
'''Returns the dataframe where missing values are imputed using MICE
Parameters:
-----------
df: pd.DataFrame
categorical: boolean, if set to True, the returned dataframe will contain the original category values
(as opposed to their integer index)
nr_iter: int, the number of imputations to be generated
Returns:
--------
df_result: pd.DataFrame where the missing values are imputed using MICE
'''
df_new = df.copy()
df_new = make_missing_np_nan(df_new)
missing, unique = imputation_heuristic_column(df, 0.99)
df_new = delete_cols(df_new, missing)
df_new = delete_cols(df_new, unique)
cat_cols, date_cols, num_cols = type_cols(df_new)
df_new = df_new[num_cols]
columns = df_new.columns
result = [0] * nr_iter
for i in range(nr_iter):
imputer = IterativeImputer(sample_posterior=True)
imputed = imputer.fit_transform(df_new)
df_imputed = pd.DataFrame(imputed, columns=columns)
result[i] = df_imputed
return result
def median_imputation(df):
'''Imputes the missing values in a data frame using median imputation
Parameters:
-----------
df: pd.DataFrame
Returns:
--------
df_result: pd.DataFrame where the missing values are imputed using the median
'''
imp_median = SimpleImputer(missing_values=np.nan, strategy='median')
df = make_missing_np_nan(df)
cat_cols, date_cols, num_cols = type_cols(df)
df_new = df[num_cols]
columns = df_new.columns
df_imputed = imp_median.fit_transform(df_new)
df_imputed = pd.DataFrame(df_imputed, columns=columns)
not_imputed_cols = cat_cols + date_cols
df_result = pd.concat([df_imputed, df[not_imputed_cols]], axis=1)
return df_result
def KNN_imputation(df, k=5):
'''Imputes the missing values in a dataframe using K-Nearest Neighbor
Parameters:
-----------
df: pd.DataFrame
k: int, number of neighboring samples to use for imputation
Returns:
--------
df_result: pd.DataFrame where the missing values are imputed using KNN
'''
df_new = df.copy()
df_new = make_missing_np_nan(df_new)
missing, unique = imputation_heuristic_column(df, 1)
df_new = delete_cols(df_new, missing)
df_new = delete_cols(df_new, unique)
cat_cols, date_cols, num_cols = type_cols(df_new)
df_new = df_new[num_cols]
columns = df_new.columns
imputer = KNNImputer(n_neighbors=k)
imputed = imputer.fit_transform(df_new)
df_imputed = pd.DataFrame(imputed, columns=columns)
not_imputed_cols = cat_cols + date_cols
df_result = pd.concat([df_imputed, df[not_imputed_cols]], axis=1)
return df_result
def make_date_season(df, date_col=None, country_col=None):
'''From a datetime variable it constructs an additional variable
containing the season number where:
winter = 0
spring = 1
summer = 2
autumn = 3
Parameters:
-----------
df: pd.DataFrame
date_col = str, containing the datetime column name
'''
if not date_col:
cat_col, date_col, num_col = type_cols(df_test)
date_col = date_col[0]
if country_col:
df['season'] = df.apply(
lambda x: get_season(x[date_col], x[country_col]), axis=1)
else:
df['season'] = df[date_col].apply(lambda x: get_season(x))
df['season'] = df['season'].apply(lambda x: season_to_num(x))
return df
def get_outliers_info(df, outlier_method):
'''For the frontend produces the outlier scores, the three sigmas, and the corresponding plot of the outliers
Parameters:
----------
df: pd.DataFrame
outlier_method: str, corresponding to one of the following outlier detection methods:
['LOF', 'IF', 'SVM', 'KNN', 'VAE']
'''
df_new = make_missing_np_nan(df)
df_new = df_new.dropna()
cat_cols, date_cols, num_cols = type_cols(df_new)
df_new = df_new[num_cols]
if outlier_method == "LOF":
outliers = detect_outliers_LOF(df_new)
elif outlier_method == "IF":
outliers = detect_outliers_IF(df_new)
elif outlier_method == "SVM":
outliers = detect_outliers_SVM(df_new)
elif outlier_method == "KNN":
outliers = detect_outliers_KNN(df_new)
elif outlier_method == "VAE":
outliers = detect_outliers_VAE(df_new)
return outliers, three_sigma(outliers), detection_scores(outliers), outlier_scores_plot(outliers)
def make_date_week_number(df, date_col=None):
'''From a datetime variable it constructs an additional variable
containing the number for the week of the year.
Parameters:
-----------
df: pd.DataFrame
date_col = list of str, containing the datetime column name
'''
if not date_col:
cat_col, date_col, num_col = type_cols(df_test)
date_col = date_col[0]
df['week_number'] = df[date_col].apply(lambda x: x.isocalendar()[1])
return df
def make_date_hour(df, date_col=None):
'''From a datetime variable it constructs an additional variable
containing the hour of the day.
Parameters:
-----------
df: pd.DataFrame
date_col = str, containing the datetime column name
'''
if not date_col:
cat_col, date_col, num_col = type_cols(df_test)
date_col = date_col[0]
df['hour'] = df[date_col].apply(lambda x: x.hour)
return df
def make_date_month(df, date_col=None):
'''From a datetime variable it constructs an additional variable
containing the number for the month of the year.
Parameters:
-----------
df: pd.DataFrame
date_col = str, containing the datetime column name
'''
if not date_col:
cat_col, date_col, num_col = type_cols(df_test)
date_col = date_col[0]
df['month'] = df[date_col].apply(lambda x: x.month)
return df
def RF_imputation(df, fast=True):
'''Returns the dataframe where missing values are imputed using Random Forest Imputation (sklearn)
ExtraTreesRegressor is used for increased speed.
Parameters:
-----------
df: pd.DataFrame
fast: boolean, if set to True, ExtraTreesRegressor is used in preference of RandomForestRegressor
Returns:
--------
df_result: pd.DataFrame where the missing values are imputed using Random Forest (MissForest)
'''
df_new = df.copy()
df_new = make_missing_np_nan(df_new)
missing, unique = imputation_heuristic_column(df, 0.99)
df_new = delete_cols(df_new, missing)
df_new = delete_cols(df_new, unique)
#categorical and datetime columns cannot be imputed, so are removed from the imputation dataframe
cat_cols, date_cols, num_cols = type_cols(df_new)
df_new = df_new[num_cols]
columns = df_new.columns
if fast:
imputer = IterativeImputer(random_state=0,
estimator=ExtraTreesRegressor(
n_estimators=10, random_state=0))
else:
imputer = IterativeImputer(random_state=0,
estimator=RandomForestRegressor(
n_estimators=10, random_state=0))
imputed = imputer.fit_transform(df_new)
df_imputed = pd.DataFrame(imputed, columns=columns)
#categorical and datetime columns are added back
not_imputed_cols = cat_cols + date_cols
df_result = pd.concat([df_imputed, df[not_imputed_cols]], axis=1)
return df_result
def weekend_or_not(df, date_col=None):
'''From a datetime variable it constructs an additional variable
indicating whether it is the weekend or not.
Parameters:
-----------
df: pd.DataFrame
date_col = str, containing the datetime column name
'''
if not date_col:
cat_col, date_col, num_col = type_cols(df_test)
date_col = date_col[0]
df = make_date_weekday(df, date_col, temp=True)
df['weekend'] = df['weekday_temp'].apply(lambda x: weekend(x))
return df.drop(['weekday_temp'], axis=1)
def make_date_weekday(df, date_col=None, temp=False):
'''From a datetime variable it constructs an additional variable
containing a number for the day of the week (monday = 0,
sunday = 6)
Parameters:
-----------
df: pd.DataFrame
date_col = str, containing the datetime column name
'''
if not date_col:
cat_col, date_col, num_col = type_cols(df_test)
date_col = date_col[0]
if temp:
df['weekday_temp'] = df[date_col].apply(lambda x: x.weekday())
else:
df['weekday'] = df[date_col].apply(lambda x: x.weekday())
return df
def DL_imputation(df, categorical=True):
'''Returns the dataframe where missing values are imputed using DataWig
Parameters:
-----------
df: pd.DataFrame
categorical: boolean, if set to True, the returned dataframe will contain the original category values
(as opposed to their integer index)
Returns:
--------
df_result: pd.DataFrame where the missing values are imputed using DataWig
'''
df_new = df.copy()
df_new = make_missing_np_nan(df_new)
missing, unique = imputation_heuristic_column(df, 0.99)
df_new = delete_cols(df_new, missing)
df_new = delete_cols(df_new, unique)
cat_cols, date_cols, num_cols = type_cols(df_new)
df_new = df_new[num_cols]
columns = df_new.columns
num_cols = [col for col in df_new.columns if is_numeric_dtype(df_new[col])]
string_cols = list(set(df_new.columns) - set(num_cols))
imputer = simple_imputer.SimpleImputer(input_columns=['1'],
output_column='2')
imputed = imputer.complete(df_new)
df_imputed = pd.DataFrame(imputed, columns=columns)
not_imputed_cols = cat_cols + date_cols
df_result = pd.concat([df_imputed, df[not_imputed_cols]], axis=1)
return df_result
def get_imputation_scores(df):
'''For the specified dataframe, each of the imputation methods is tried on the observations without missing values and synthethically imputed
to establish the best performing imputation method.
Parameters:
-----------
df: pd.DataFrame
Returns:
result: dict containing for each of the imputation techniques (mean, median, multiple, KNN and RF) the NRMSE
'''
cat_cols, date_cols, num_cols = type_cols(df)
df_new = df[num_cols]
if df.shape[0] * df.shape[1] < 10000:
imputations = ['mean', 'median', 'regression', 'KNN', 'RF']
else:
imputations = ['mean', 'median', 'regression', 'RF']
result = {}
for imputation in imputations:
score = imputation_error(df_new, imputation=imputation)
result[imputation] = score
return result