def predict(data): # extract first letter from cabin pf.extract_cabin_letter(data, 'cabin') # impute NA categorical for var in ['age', 'fare']: pf.add_missing_indicator(data, var) # impute NA numerical for var in config.CATEGORICAL_VARS: pf.impute_na(data, var) # Group rare labels for var in config.CATEGORICAL_VARS: pf.remove_rare_labels(data, var, config.FREQUENT_LABELS) # encode variables data = pf.encode_categorical(df, config.CATEGORICAL_VARS) # scale variables data = pf.scale_features(data, config.OUTPUT_SCALER_PATH) # make predictions predictions, _ = pf.predict(data, config.OUTPUT_MODEL_PATH) return predictions
def predict(data): # extract first letter from cabin pf.extract_cabin_letter(data, 'cabin') # impute NA numerical for var in config.NUMERICAL_TO_IMPUTE: pf.add_missing_indicator(data, var) pf.impute_na(data, var, config.IMPUTATION_DICT[var]) # impute NA categorical for var in config.CATEGORICAL_VARS: pf.impute_na(data, var) # Group rare labels for var, labels in config.FREQUENT_LABELS.items(): pf.remove_rare_labels(data, var, labels) # encode variables for var in config.CATEGORICAL_VARS: data = pf.encode_categorical(data, var) # check all dummies were added pf.check_dummy_variables(data, config.DUMMY_VARIABLES) data = data[config.ALL_VARS] # scale variables data = pf.scale_features(data, config.OUTPUT_SCALER_PATH) # make predictions predictions = pf.predict(data, config.OUTPUT_MODEL_PATH) return predictions
def predict(data): # extract first letter from cabin data[config.EXTRACT_VARIABLE] = pf.extract_cabin_letter( data, config.EXTRACT_VARIABLE) # impute NA categorical for var in config.CATEGORICAL_TO_ENCODE: data[var] = pf.impute_na(data, var, replacement='Missing') # impute NA numerical for var in config.NUMERICAL_TO_IMPUTE: if (var == 'age'): data[var] = pf.add_missing_indicator(data, var, config.AGE_MEDIAN) else: data[var] = pf.add_missing_indicator(data, var, config.FARE_MEDIAN) # Group rare labels for var in config.CATEGORICAL_TO_ENCODE: data[var] = pf.remove_rare_labels(data, var, config.RARE_VALUE) # encode variables for var in config.CATEGORICAL_TO_ENCODE: data = pf.encode_categorical(data, var) # check all dummies were added pf.check_dummy_variables(data, config.DUMMY_VARIABLE) # scale variables data = pf.scale_features(data[config.FEATURES], config.OUTPUT_SCALER_PATH) # make predictions predictions = pf.predict(data, config.OUTPUT_MODEL_PATH) return predictions
def predict(data): # impute categorical variables for var in config.CATEGORICAL_VARS: data[var] = pf.impute_na(data, var, replacement='Missing') # impute numerical variables for var in config.NUMERICAL_TO_IMPUTE: # add missing indicator first data[var + '_na'] = pf.add_missing_indicator(data, var) # impute NA data[var] = pf.impute_na(data, var, replacement=config.IMPUTATION_DICT[var]) # Group rare labels for var in config.CATEGORICAL_VARS: data[var] = pf.remove_rare_labels(data, var, config.FREQUENT_LABELS[var]) # encode categorical variables for var in config.CATEGORICAL_VARS: data = pf.encode_categorical(data, var) # check all dummies were added data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES) # scale variables scaler = pf.scale_features(data, config.OUTPUT_SCALER_PATH) # make predictions predictions = pf.predict(data, config.OUTPUT_MODEL_PATH) return predictions
def predict(data): # extract first letter from cabin X_test = pf.extract_cabin_letter(data, config.IMPUTATION_DICT['cabin_variable']) # impute NA categorical X_test = pf.add_missing_indicator(X_test, config.CATEGORICAL_VARS) # impute NA numerical for var in config.NUMERICAL_TO_IMPUTE: X_test = pf.impute_na(X_test,var,replace_by=config.IMPUTATION_DICT[var], add_na_columns=True) # Group rare labels X_test = pf.remove_rare_labels(X_test, config.FREQUENT_LABELS) # encode variables for var in config.CATEGORICAL_VARS: X_test = pf.encode_categorical(X_test, var) X_test.drop(labels=config.CATEGORICAL_VARS, axis=1, inplace=True) # check all dummies were added X_test = pf.check_dummy_variables(X_test, config.DUMMY_VARIABLES) # scale variables X_test = pf.scale_features(X_test, config.OUTPUT_SCALER_PATH) # make predictions predictions = pf.predict(X_test,config.OUTPUT_MODEL_PATH) return predictions
def predict(data): # extract first letter from cabin data["cabin"] = pf.extract_cabin_letter(data, "cabin") # impute NA categorical for var in config.CATEGORICAL_VARS: data[var] = pf.impute_na(data, var, replacement="Missing") # impute NA numerical for var in config.NUMERICAL_TO_IMPUTE: data[var + '_NA'] = pf.add_missing_indicator(data, var) data[var] = pf.impute_na(data, var, replacement=config.IMPUTATION_DICT[var]) # Group rare labels for var in config.CATEGORICAL_VARS: data[var] = pf.remove_rare_labels(data, var, config.FREQUENT_LABELS[var]) # encode variables for var in config.CATEGORICAL_VARS: data = pf.encode_categorical(data, var) # check all dummies were added data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES) # scale variables data = pf.scale_features(data, config.OUTPUT_SCALER_PATH) # make predictions predictions = pf.predict(data, config.OUTPUT_MODEL_PATH) return predictions
def predict(data): data = pf.load_data(config.PATH_TO_DATASET) X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET) data = X_test.copy() # impute categorical variables data = pf.add_missing_indicator(data, config.CATEGORICAL_VARS) # extract first letter from cabin data = pf.extract_cabin_letter(data, 'cabin') # impute NA categorical data = pf.impute_na(data, config.CATEGORICAL_VARS) # impute NA numerical data = pf.add_missing_indicator(data, config.NUMERICAL_TO_IMPUTE) data = pf.impute_num(data, config.NUMERICAL_TO_IMPUTE) # Group rare labels data = pf.remove_rare_labels(data, config.CATEGORICAL_VARS) # encode variables data, data_features = pf.encode_categorical(data, config.CATEGORICAL_VARS) print(data.head(1)) # check all dummies were added data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES) # scale variables data = pf.scale_features(data, config.OUTPUT_SCALER_PATH) # make predictions class_, pred = pf.predict(data, config.OUTPUT_MODEL_PATH) return class_
def predict(data): # extract first letter from cabin data['cabin'] = pf.extract_cabin_letter(data, 'cabin') # impute NA categorical for var in config.CATEGORICAL_VARS: data[var] = pf.impute_na(data, var, value='Missing') # impute NA numerical for var in config.NUMERICAL_TO_IMPUTE: # add missing indicator pf.add_missing_indicator(data, var) # replace NaN by median median_val = data[var].median() data[var] = pf.impute_na(data, var, value=median_val) # Group rare labels for var in config.CATEGORICAL_VARS: data[var] = pf.remove_rare_labels(data, var, config.FREQUENT_LABELS[var]) # encode variables for var in config.CATEGORICAL_VARS: data = pf.encode_categorical(data, var) # check all dummies were added for var in config.CATEGORICAL_VARS: pf.check_dummy_variables(data, config.DUMMY_VARIABLES[var]) # scale variables data = pf.scale_features(data, config.OUTPUT_SCALER_PATH) # make predictions predictions = pf.predict(data, config.OUTPUT_MODEL_PATH) return predictions
def predict(data): # extract first letter from cabin data = pf.extract_cabin_letter(data, 'cabin') # impute NA categorical for var in config.CATEGORICAL_VARS: data = pf.impute_na(data, var, config.IMPUTATION_DICT) # impute NA numerical for var in ['age', 'fare']: data = pf.impute_na(data, var, config.IMPUTATION_DICT) # add indicator variables for var in ['age', 'fare']: data = pf.add_missing_indicator(data, var) # Group rare labels for var in config.CATEGORICAL_VARS: data = pf.remove_rare_labels(data, config.FREQUENT_LABELS, var) # encode variables for var in config.CATEGORICAL_VARS: data = pf.encode_categorical(data, var) # check all dummies were added data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES) # scale variables data = pf.scale_features(data, config.ORDERED_COLUMNS, config.OUTPUT_SCALER_PATH) # make predictions predictions = pf.predict(data, config.ORDERED_COLUMNS, config.OUTPUT_MODEL_PATH) return predictions
def predict(data): # extract first letter from cabin data['cabin'] = pf.extract_cabin_letter(data, 'cabin') # impute NA categorical for var in config.CATEGORICAL_VARS: data[var] = pf.impute_na(data, var, replacement='Missing') # impute NA numerical for var in config.NUMERICAL_TO_IMPUTE: data[var] = pf.add_missing_indicator(data, var) data[var] = pf.impute_na(data, var, config.IMPUTATION_DICT[var]) # Group rare labels for var in config.CATEGORICAL_VARS: data[var] = pf.remove_rare_labels(data, var, config.FREQUENT_LABELS[var]) # encode variables data = pf.encode_categorical(data, config.CATEGORICAL_VARS) # check all dummies were added # print(data.shape) # print(data.columns) data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES) # print(data.columns) # print(data.shape) data = data[config.FEATURES] # scale variables data = pf.scale_features(data, config.OUTPUT_SCALER_PATH) # print('test data shape', data.shape) # make predictions predictions = pf.predict(data, config.OUTPUT_MODEL_PATH) return predictions
df_data = pf.load_data(config.PATH_TO_DATASET) df_target = df_data[config.TARGET] df_data = df_data.drop([config.TARGET], axis=1) # divide data set X_train, X_test, y_train, y_test = pf.divide_train_test( df_data, df_target, seed=config.GLOBAL_SEED) # get first letter from cabin variable X_train = pf.extract_cabin_letter(X_train, config.IMPUTATION_DICT['cabin_variable']) X_test = pf.extract_cabin_letter(X_test, config.IMPUTATION_DICT['cabin_variable']) # impute categorical variables X_train = pf.add_missing_indicator(X_train, config.CATEGORICAL_VARS) X_test = pf.add_missing_indicator(X_test, config.CATEGORICAL_VARS) # impute numerical variable for var in config.NUMERICAL_TO_IMPUTE: X_train = pf.impute_na(X_train, var, replace_by=config.IMPUTATION_DICT[var], add_na_columns=True) X_test = pf.impute_na(X_test, var, replace_by=config.IMPUTATION_DICT[var], add_na_columns=True) # Group rare labels X_train = pf.remove_rare_labels(X_train, config.FREQUENT_LABELS)
# ================================================ # TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL # Load data data = pf.load_data(config.PATH_TO_DATASET) # divide data set X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET) # get first letter from cabin variable X_train['cabin'] = pf.extract_cabin_letter(X_train, 'cabin') X_test['cabin'] = pf.extract_cabin_letter(X_test, 'cabin') # impute categorical variables for var in config.CATEGORICAL_VARS: X_train[var + '_na'] = pf.add_missing_indicator(X_train, var) X_train[var] = pf.impute_na(X_train, var) X_test[var + '_na'] = pf.add_missing_indicator(X_test, var) X_test[var] = pf.impute_na(X_test, var) # impute numerical variable for var in config.NUMERICAL_TO_IMPUTE: X_train[var + '_na'] = pf.add_missing_indicator(X_train, var) X_train[var] = pf.impute_na(X_train, var, config.IMPUTATION_DICT[var]) X_test[var + '_na'] = pf.add_missing_indicator(X_test, var) X_test[var] = pf.impute_na(X_test, var, config.IMPUTATION_DICT[var]) # Group rare labels for var in config.CATEGORICAL_VARS: X_train[var] = pf.remove_rare_labels(X_train, var,
# divide data set X_train, X_test, y_train, y_test = pf.divide_train_test(df, config.TARGET) # get first letter from cabin variable X_train['cabin'] = pf.extract_cabin_letter(X_train, 'cabin') # impute categorical variables for var in config.CATEGORICAL_VARS: X_train[var] = pf.impute_na(X_train, var) # impute numerical variable for var in config.NUMERICAL_TO_IMPUTE: # add missing indicator X_train[var + '_NA'] = pf.add_missing_indicator(X_train, var) # replace NaN by median X_train[var] = pf.impute_na(X_train, var, replacement=config.IMPUTATION_DICT[var]) # Group rare labels for var in config.CATEGORICAL_VARS: X_train[var] = pf.remove_rare_labels(X_train, var, config.FREQUENT_LABELS[var]) # encode categorical variables for var in config.CATEGORICAL_VARS: X_train = pf.encode_categorical(X_train, var)
data = pf.load_data(config.PATH_TO_DATASET) # divide data set X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET) # get first letter from cabin variable X_train['cabin'] = pf.extract_cabin_letter(X_train, 'cabin') # impute categorical variables for var in config.CATEGORICAL_VARS: X_train[var] = pf.impute_na(X_train, var, value='Missing') # impute numerical variable for var in config.NUMERICAL_TO_IMPUTE: # add missing indicator pf.add_missing_indicator(X_train, var) # replace NaN by median median_val = X_train[var].median() X_train[var] = pf.impute_na(X_train, var, value=median_val) # Group rare labels for var in config.CATEGORICAL_VARS: X_train[var] = pf.remove_rare_labels(X_train, var, config.FREQUENT_LABELS[var]) # encode categorical variables for var in config.CATEGORICAL_VARS: X_train = pf.encode_categorical(X_train, var) # check all dummies were added
X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET) # get first letter from cabin variable X_train[config.EXTRACT_VARIABLE] = pf.extract_cabin_letter(X_train, config.EXTRACT_VARIABLE) # impute categorical variables for var in config.CATEGORICAL_TO_ENCODE: X_train[var] = pf.impute_na(X_train, var, replacement='Missing') # impute numerical variable for var in config.NUMERICAL_TO_IMPUTE: if (var == 'age'): X_train[var] = pf.add_missing_indicator(X_train, var, config.AGE_MEDIAN) else: X_train[var] = pf.add_missing_indicator(X_train, var, config.FARE_MEDIAN) # Group rare labels for var in config.CATEGORICAL_TO_ENCODE: X_train[var] = pf.remove_rare_labels(X_train, var, config.RARE_VALUE) # encode categorical variables for var in config.CATEGORICAL_TO_ENCODE: X_train = pf.encode_categorical(X_train, var) # check all dummies were added
import config # ================================================ # TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL # Load data df = pf.load_data(config.PATH_TO_DATASET) # divide data set X_train, X_test, y_train, y_test = pf.divide_train_test(df, config.TARGET) # get first letter from cabin variable X_train = pf.extract_cabin_letter(X_train, 'cabin') # impute categorical variables X_train = pf.add_missing_indicator(X_train, config.CATEGORICAL_VARS) X_train = pf.impute_na(X_train, config.CATEGORICAL_VARS) # impute numerical variable X_train = pf.add_missing_indicator(X_train, config.NUMERICAL_TO_IMPUTE) X_train = pf.impute_num(X_train, config.NUMERICAL_TO_IMPUTE) # Group rare labels X_train = pf.remove_rare_labels(X_train, config.CATEGORICAL_VARS) # encode categorical variables X_train, X_train_features = pf.encode_categorical(X_train, config.CATEGORICAL_VARS) # check dummy variables X_check
# divide data set X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET) # get first letter from cabin variable X_train['cabin'] = pf.extract_cabin_letter(df=X_train, var='cabin') X_test['cabin'] = pf.extract_cabin_letter(df=X_test, var='cabin') # impute categorical variables for var in config.CATEGORICAL_VARS: X_train[var] = pf.impute_na(X_train, var, replacement='Missing') X_test[var] = pf.impute_na(X_test, var, replacement='Missing') # impute numerical variable for var in config.NUMERICAL_TO_IMPUTE: X_train[var] = pf.add_missing_indicator(df=X_train, var=var) X_test[var] = pf.add_missing_indicator(df=X_test, var=var) # Group rare labels for var in config.CATEGORICAL_VARS: X_train[var] = pf.remove_rare_labels(X_train, var, config.FREQUENT_LABELS[var]) X_test[var] = pf.remove_rare_labels(X_test, var, config.FREQUENT_LABELS[var]) # encode categorical variables X_train = pf.encode_categorical(df=X_train, var=config.CATEGORICAL_VARS) X_test = pf.encode_categorical(df=X_test, var=config.CATEGORICAL_VARS) # check all dummies were added X_train = pf.check_dummy_variables(X_train, config.DUMMY_VARIABLES)
data = pf.load_data(config.PATH_TO_DATASET) # divide data set X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET) # get first letter from cabin variable X_train["cabin"] = pf.extract_cabin_letter(X_train, "cabin") print(X_train["cabin"].unique()) # impute categorical variables for var in config.CATEGORICAL_VARS: X_train[var] = pf.impute_na(X_train, var, replacement='Missing') # impute numerical variable with median for var in config.NUMERICAL_TO_IMPUTE: X_train[var + "_na"] = pf.add_missing_indicator(X_train, var) median_train_var = config.IMPUTATION_DICT[var] X_train[var] = pf.impute_na(X_train, var, replacement=median_train_var) # Group rare labels for var in config.CATEGORICAL_VARS: # Frequent labels found in Train set freq_labels = config.FREQUENT_LABELS[var] # Remove rare labels from both train and test set X_train[var] = pf.remove_rare_labels(X_train, var, freq_labels) # encode categorical variables for var in config.CATEGORICAL_VARS: X_train = pf.encode_categorical(X_train, var) # check all dummies were added
df = pf.load_data(config.PATH_TO_DATASET) # divide data set X_train, X_test, y_train, y_test = pf.divide_train_test(df, config.TARGET) # get first letter from cabin variable X_train = pf.extract_cabin_letter(X_train, 'cabin') X_test = pf.extract_cabin_letter(X_test, 'cabin') # impute categorical variables X_train = pf.impute_na(X_train, config.CATEGORICAL_VARS) X_test = pf.impute_na(X_test, config.CATEGORICAL_VARS) # impute numerical variable for var in config.IMPUTATION_DICT.keys(): X_train = pf.add_missing_indicator(X_train, var) X_test = pf.add_missing_indicator(X_test, var) X_train = pf.impute_na(X_train, var, config.IMPUTATION_DICT[var]) X_test = pf.impute_na(X_test, var, config.IMPUTATION_DICT[var]) # Group rare labels for var in config.FREQUENT_LABELS.keys(): X_train = pf.remove_rare_labels(X_train, var, config.FREQUENT_LABELS[var]) X_test = pf.remove_rare_labels(X_test, var, config.FREQUENT_LABELS[var]) # encode categorical variables for var in config.CATEGORICAL_VARS: X_train = pf.encode_categorical(X_train, var) X_test = pf.encode_categorical(X_test, var)