def predict(data):
# extract first letter from cabin
pf.extract_cabin_letter(data, 'cabin')
# impute NA categorical
for var in ['age', 'fare']:
pf.add_missing_indicator(data, var)
# impute NA numerical
for var in config.CATEGORICAL_VARS:
pf.impute_na(data, var)
# Group rare labels
for var in config.CATEGORICAL_VARS:
pf.remove_rare_labels(data, var, config.FREQUENT_LABELS)
# encode variables
data = pf.encode_categorical(df, config.CATEGORICAL_VARS)
# scale variables
data = pf.scale_features(data, config.OUTPUT_SCALER_PATH)
# make predictions
predictions, _ = pf.predict(data, config.OUTPUT_MODEL_PATH)
return predictions
def predict(data):
# extract first letter from cabin
pf.extract_cabin_letter(data, 'cabin')
# impute NA numerical
for var in config.NUMERICAL_TO_IMPUTE:
pf.add_missing_indicator(data, var)
pf.impute_na(data, var, config.IMPUTATION_DICT[var])
# impute NA categorical
for var in config.CATEGORICAL_VARS:
pf.impute_na(data, var)
# Group rare labels
for var, labels in config.FREQUENT_LABELS.items():
pf.remove_rare_labels(data, var, labels)
# encode variables
for var in config.CATEGORICAL_VARS:
data = pf.encode_categorical(data, var)
# check all dummies were added
pf.check_dummy_variables(data, config.DUMMY_VARIABLES)
data = data[config.ALL_VARS]
# scale variables
data = pf.scale_features(data, config.OUTPUT_SCALER_PATH)
# make predictions
predictions = pf.predict(data, config.OUTPUT_MODEL_PATH)
return predictions
def predict(data):
# extract first letter from cabin
data[config.EXTRACT_VARIABLE] = pf.extract_cabin_letter(
data, config.EXTRACT_VARIABLE)
# impute NA categorical
for var in config.CATEGORICAL_TO_ENCODE:
data[var] = pf.impute_na(data, var, replacement='Missing')
# impute NA numerical
for var in config.NUMERICAL_TO_IMPUTE:
if (var == 'age'):
data[var] = pf.add_missing_indicator(data, var, config.AGE_MEDIAN)
else:
data[var] = pf.add_missing_indicator(data, var, config.FARE_MEDIAN)
# Group rare labels
for var in config.CATEGORICAL_TO_ENCODE:
data[var] = pf.remove_rare_labels(data, var, config.RARE_VALUE)
# encode variables
for var in config.CATEGORICAL_TO_ENCODE:
data = pf.encode_categorical(data, var)
# check all dummies were added
pf.check_dummy_variables(data, config.DUMMY_VARIABLE)
# scale variables
data = pf.scale_features(data[config.FEATURES], config.OUTPUT_SCALER_PATH)
# make predictions
predictions = pf.predict(data, config.OUTPUT_MODEL_PATH)
return predictions
def predict(data):
# extract first letter from cabin
X_test = pf.extract_cabin_letter(data, config.IMPUTATION_DICT['cabin_variable'])
# impute NA categorical
X_test = pf.add_missing_indicator(X_test, config.CATEGORICAL_VARS)
# impute NA numerical
for var in config.NUMERICAL_TO_IMPUTE:
X_test = pf.impute_na(X_test,var,replace_by=config.IMPUTATION_DICT[var], add_na_columns=True)
# Group rare labels
X_test = pf.remove_rare_labels(X_test, config.FREQUENT_LABELS)
# encode variables
for var in config.CATEGORICAL_VARS:
X_test = pf.encode_categorical(X_test, var)
X_test.drop(labels=config.CATEGORICAL_VARS, axis=1, inplace=True)
# check all dummies were added
X_test = pf.check_dummy_variables(X_test, config.DUMMY_VARIABLES)
# scale variables
X_test = pf.scale_features(X_test, config.OUTPUT_SCALER_PATH)
# make predictions
predictions = pf.predict(X_test,config.OUTPUT_MODEL_PATH)
return predictions
def predict(data):
# extract first letter from cabin
data["cabin"] = pf.extract_cabin_letter(data, "cabin")
# impute NA categorical
for var in config.CATEGORICAL_VARS:
data[var] = pf.impute_na(data, var, replacement="Missing")
# impute NA numerical
for var in config.NUMERICAL_TO_IMPUTE:
median_val = data[var].median()
data[var] = pf.impute_na(data, var, replacement=median_val)
# Group rare labels
for var in config.CATEGORICAL_VARS:
data[var] = pf.remove_rare_labels(data, var, config.FREQUENT_LABELS[var])
# encode variables
for var in config.CATEGORICAL_VARS:
data = pf.encode_categorical(data, var)
# check all dummies were added
data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES)
# scale variables
data = pf.scale_features(data, config.OUTPUT_SCALER_PATH)
# make predictions
predictions = pf.predict(data, config.OUTPUT_MODEL_PATH)
return predictions
def predict(data):
# extract first letter from cabin
data['cabin'] = pf.extract_cabin_letter(data, 'cabin')
# impute NA categorical
data[config.CATEGORICAL_VARS] = pf.impute_na(data[config.CATEGORICAL_VARS],
'Missing')
# impute NA numerical
data[config.NUMERICAL_TO_IMPUTE] = pf.impute_na(
data[config.NUMERICAL_TO_IMPUTE], 'Numerical')
# Group rare labels
for var in config.CATEGORICAL_VARS:
data[var] = pf.remove_rare_labels(data, var,
config.FREQUENT_LABELS[var])
# encode variables
data = pf.encode_categorical(data, config.CATEGORICAL_VARS)
print(data.shape)
# check all dummies were added
data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES)
print(data.shape)
# scale variables
data = pf.scale_features(data, config.OUTPUT_SCALER_PATH)
# make predictions
predictions = pf.predict(data, config.OUTPUT_MODEL_PATH)
return predictions
def predict(data):
# extract first letter from cabin
data['cabin'] = pf.extract_cabin_letter(data, var='cabin')
# impute NA categorical
for var in config.CATEGORICAL_VARS:
data[var] = pf.impute_na(data, var, replacement='Missing')
# impute NA numerical
for var in config.NUMERICAL_TO_IMPUTE:
data[var] = pf.add_missing_indicator(data, var)
# Group rare labels
for var in config.CATEGORICAL_VARS:
data[var] = pf.remove_rare_labels(data, var, frequent_labels = config.FREQUENT_LABELS[var])
# encode variables
data = pf.encode_categorical(data, var=config.CATEGORICAL_VARS)
# check all dummies were added
data = pf.check_dummy_variables(data, dummy_list = config.DUMMY_VARIABLES)
# scale variables
data = pf.scale_features(data[config.FEATURES],
config.OUTPUT_SCALER_PATH)
# make predictions
predictions = pf.predict(data, config.OUTPUT_MODEL_PATH)
return predictions
def predict(data):
# extract first letter from cabin
data["cabin"] = pf.extract_cabin_letter(data, "cabin")
# impute NA categorical
for var in config.CATEGORICAL_VARS:
data[var] = pf.impute_na(data, var, replacement='Missing')
# impute NA numerical
for var in config.NUMERICAL_TO_IMPUTE:
data[var + "_na"] = pf.add_missing_indicator(data, var)
median_train_var = config.IMPUTATION_DICT[var]
data[var] = pf.impute_na(data, var, replacement=median_train_var)
# Group rare labels
for var in config.CATEGORICAL_VARS:
freq_labels = config.FREQUENT_LABELS[var]
# Remove rare labels from both train and test set
data[var] = pf.remove_rare_labels(data, var, freq_labels)
# encode variables
for var in config.CATEGORICAL_VARS:
data = pf.encode_categorical(data, var)
# check all dummies were added
data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES)
# scale variables
data = pf.scale_features(data, config.OUTPUT_SCALER_PATH)
# make predictions
predictions = pf.predict(data, config.OUTPUT_MODEL_PATH)
return predictions
def predict(data):
# extract first letter from cabin
data = pf.extract_cabin_letter(data, 'cabin')
# impute NA categorical
for var in config.CATEGORICAL_VARS:
data = pf.impute_na(data, var, config.IMPUTATION_DICT)
# impute NA numerical
for var in ['age', 'fare']:
data = pf.impute_na(data, var, config.IMPUTATION_DICT)
# add indicator variables
for var in ['age', 'fare']:
data = pf.add_missing_indicator(data, var)
# Group rare labels
for var in config.CATEGORICAL_VARS:
data = pf.remove_rare_labels(data, config.FREQUENT_LABELS, var)
# encode variables
for var in config.CATEGORICAL_VARS:
data = pf.encode_categorical(data, var)
# check all dummies were added
data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES)
# scale variables
data = pf.scale_features(data, config.ORDERED_COLUMNS, config.OUTPUT_SCALER_PATH)
# make predictions
predictions = pf.predict(data, config.ORDERED_COLUMNS, config.OUTPUT_MODEL_PATH)
return predictions
def predict(data):
data = pf.load_data(config.PATH_TO_DATASET)
X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET)
data = X_test.copy()
# impute categorical variables
data = pf.add_missing_indicator(data, config.CATEGORICAL_VARS)
# extract first letter from cabin
data = pf.extract_cabin_letter(data, 'cabin')
# impute NA categorical
data = pf.impute_na(data, config.CATEGORICAL_VARS)
# impute NA numerical
data = pf.add_missing_indicator(data, config.NUMERICAL_TO_IMPUTE)
data = pf.impute_num(data, config.NUMERICAL_TO_IMPUTE)
# Group rare labels
data = pf.remove_rare_labels(data, config.CATEGORICAL_VARS)
# encode variables
data, data_features = pf.encode_categorical(data, config.CATEGORICAL_VARS)
print(data.head(1))
# check all dummies were added
data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES)
# scale variables
data = pf.scale_features(data, config.OUTPUT_SCALER_PATH)
# make predictions
class_, pred = pf.predict(data, config.OUTPUT_MODEL_PATH)
return class_
def predict(data):
# extract first letter from cabin
data['cabin'] = pf.extract_cabin_letter(data, 'cabin')
# impute NA categorical
for el in config.CATEGORICAL_VARS:
data[el] = pf.impute_na(data, el, replacement='Missing')
# impute NA numerical
for el in config.NUMERICAL_TO_IMPUTE:
# add missing indicator first
data[el + '_NA'] = pf.add_missing_indicator(data, el)
# impute NA
data[el] = pf.impute_na(data,
el,
replacement=config.IMPUTATION_DICT[el])
# Group rare labels
for el in config.CATEGORICAL_VARS:
data[el] = pf.remove_rare_labels(data, el, config.FREQUENT_LABELS[el])
# encode variables
for el in config.CATEGORICAL_VARS:
data = pf.encode_categorical(data, el)
# check all dummies were added
data = pf.check_dummy_variables(data, config.DUMMY_VARIABLES)
# scale variables
data = pf.scale_features(data, config.OUTPUT_SCALER_PATH)
# make predictions
predictions = pf.predict(data, config.OUTPUT_MODEL_PATH)
return predictions
def predict(data):
# extract first letter from cabin
data = pf.extract_cabin_letter(data, 'cabin')
# impute NA categorical
for var in config.CATEGORICAL_VARS:
data[var] = pf.impute_na(data, var, replacement='Missing')
# impute NA numerical
for var in config.NUMERICAL_TO_IMPUTE:
data[var] = pf.impute_na(data,
var,
replacement=config.IMPUTATION_DICT[var])
# Group rare labels
for var in config.CATEGORICAL_VARS:
data[var] = pf.remove_rare_labels(data, var,
config.FREQUENT_LABELS[var])
# encode variables
for var in config.CATEGORICAL_VARS:
data = pf.encode_categorical(data, var)
# check all dummies were added
for var in config.DUMMY_VARIABLES:
if (var not in X_train.columns):
data[var] = 0
# scale variables
data = pf.scale_features(data, config.OUTPUT_SCALER_PATH)
# make predictions
predictions = pf.predict(data, config.OUTPUT_MODEL_PATH)
return predictions
import pandas as pd
import config
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
df_data = pf.load_data(config.PATH_TO_DATASET)
df_target = df_data[config.TARGET]
df_data = df_data.drop([config.TARGET], axis=1)
# divide data set
X_train, X_test, y_train, y_test = pf.divide_train_test(
df_data, df_target, seed=config.GLOBAL_SEED)
# get first letter from cabin variable
X_train = pf.extract_cabin_letter(X_train,
config.IMPUTATION_DICT['cabin_variable'])
X_test = pf.extract_cabin_letter(X_test,
config.IMPUTATION_DICT['cabin_variable'])
# impute categorical variables
X_train = pf.add_missing_indicator(X_train, config.CATEGORICAL_VARS)
X_test = pf.add_missing_indicator(X_test, config.CATEGORICAL_VARS)
# impute numerical variable
for var in config.NUMERICAL_TO_IMPUTE:
X_train = pf.impute_na(X_train,
var,
replace_by=config.IMPUTATION_DICT[var],
add_na_columns=True)
X_test = pf.impute_na(X_test,
var,
import preprocessing_functions as pf
import config
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
data = pf.load_data(config.PATH_TO_DATASET)
# divide data set
X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET)
# get first letter from cabin variable
X_train['cabin'] = pf.extract_cabin_letter(X_train, 'cabin')
X_test['cabin'] = pf.extract_cabin_letter(X_test, 'cabin')
# impute categorical variables
for var in config.CATEGORICAL_VARS:
X_train[var + '_na'] = pf.add_missing_indicator(X_train, var)
X_train[var] = pf.impute_na(X_train, var)
X_test[var + '_na'] = pf.add_missing_indicator(X_test, var)
X_test[var] = pf.impute_na(X_test, var)
# impute numerical variable
for var in config.NUMERICAL_TO_IMPUTE:
X_train[var + '_na'] = pf.add_missing_indicator(X_train, var)
X_train[var] = pf.impute_na(X_train, var, config.IMPUTATION_DICT[var])
X_test[var + '_na'] = pf.add_missing_indicator(X_test, var)
X_test[var] = pf.impute_na(X_test, var, config.IMPUTATION_DICT[var])
import preprocessing_functions as pf
import config
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
data = pf.load_data(config.PATH_TO_DATASET)
# divide data set
X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET)
# get first letter from cabin variable
X_train['cabin'] = pf.extract_cabin_letter(X_train)
# impute categorical variables
for var in config.CATEGORICAL_VARS:
X_train[var] = pf.impute_na(X_train, var)
# impute numerical variable
for var in config.NUMERICAL_TO_IMPUTE:
X_train = pf.add_missing_indicator(X_train, var)
X_train[var] = pf.impute_na(X_train, var, config.IMPUTATION_DICT[var])
# Group rare labels
for var in config.CATEGORICAL_VARS:
import preprocessing_functions as pf
import config
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
data = pf.load_data(config.PATH_TO_DATASET)
# divide data set
X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET)
# get first letter from cabin variable
X_train[config.EXTRACT_VARIABLE] = pf.extract_cabin_letter(X_train, config.EXTRACT_VARIABLE)
# impute categorical variables
for var in config.CATEGORICAL_TO_ENCODE:
X_train[var] = pf.impute_na(X_train, var, replacement='Missing')
# impute numerical variable
for var in config.NUMERICAL_TO_IMPUTE:
if (var == 'age'):
X_train[var] = pf.add_missing_indicator(X_train, var, config.AGE_MEDIAN)
else:
X_train[var] = pf.add_missing_indicator(X_train, var, config.FARE_MEDIAN)
import preprocessing_functions as pf
import config
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
df = pf.load_data(config.PATH_TO_DATASET)
# divide data set
xtrain, xtest, ytrain, ytest = pf.divide_train_test(df, config.TARGET)
# # get first letter from cabin variable
xtrain['cabin'] = pf.extract_cabin_letter(xtrain, 'cabin')
# # impute categorical variables
xtrain[config.CATEGORICAL_VARS] = pf.impute_na(xtrain[config.CATEGORICAL_VARS],
'Missing')
# # impute numerical variable
xtrain[config.NUMERICAL_TO_IMPUTE] = pf.impute_na(
xtrain[config.NUMERICAL_TO_IMPUTE], 'Numerical')
# # Group rare labels
for var in config.CATEGORICAL_VARS:
xtrain[var] = pf.remove_rare_labels(xtrain, var,
config.FREQUENT_LABELS[var])
# # encode categorical variables
xtrain = pf.encode_categorical(xtrain, config.CATEGORICAL_VARS)
import config
import preprocessing_functions as pf
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
data = pf.load_data(config.PATH_TO_DATASET)
# divide data set
X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET)
# get first letter from cabin variable
X_train.loc[:, "cabin"] = pf.extract_cabin_letter(X_train, "cabin")
# impute categorical variables
for var in config.CATEGORICAL_VARS:
X_train.loc[:, var] = pf.impute_na(X_train, var, replacement="Missing")
# impute numerical variable
for var in config.NUMERICAL_TO_IMPUTE:
median_val = X_train[var].median()
X_train.loc[:, var] = pf.impute_na(X_train, var, replacement=median_val)
# Group rare labels
for var in config.CATEGORICAL_VARS:
X_train.loc[:, var] = pf.remove_rare_labels(X_train, var,
config.FREQUENT_LABELS[var])
# encode categorical variables
for var in config.CATEGORICAL_VARS:
import preprocessing_functions as pf
import config
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
data = pf.load_data(config.PATH_TO_DATASET)
# divide data set
X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET)
# get first letter from cabin variable
data = pf.extract_cabin_letter(data, 'cabin')
# impute categorical variables
for var in config.CATEGORICAL_VARS:
X_train[var] = pf.impute_na(X_train, var, replacement='Missing')
# impute numerical variable
for var in config.NUMERICAL_TO_IMPUTE:
X_train[var] = pf.impute_na(X_train,
var,
replacement=config.IMPUTATION_DICT[var])
# Group rare labels
for var in config.CATEGORICAL_VARS:
X_train[var] = pf.remove_rare_labels(X_train, var,
config.FREQUENT_LABELS[var])
# encode categorical variables
import preprocessing_functions as pf
import config
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
df = pf.load_data(config.PATH_TO_DATASET)
# divide data set
X_train, X_test, y_train, y_test = pf.divide_train_test(df, config.TARGET)
# get first letter from cabin variable
pf.extract_cabin_letter(X_train, 'cabin')
# impute numerical variable
for var in config.NUMERICAL_TO_IMPUTE:
pf.add_missing_indicator(X_train, var)
pf.impute_na(X_train, var, config.IMPUTATION_DICT[var])
# impute categorical variables
for var in config.CATEGORICAL_VARS:
pf.impute_na(X_train, var)
# Group rare labels
for var, labels in config.FREQUENT_LABELS.items():
pf.remove_rare_labels(X_train, var, labels)
# encode categorical variables
for var in config.CATEGORICAL_VARS:
X_train = pf.encode_categorical(X_train, var)
import config
import warnings
warnings.simplefilter(action='ignore')
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
data = pf.load_data(config.PATH_TO_DATASET)
# divide data set
X_train, _, y_train, _ = pf.divide_train_test(data, config.TARGET)
# get first letter from cabin variable
X_train["cabin"] = pf.extract_cabin_letter(X_train, "cabin")
# impute categorical variables
for var in config.CATEGORICAL_VARS:
X_train[var] = pf.impute_na(X_train, var)
# impute numerical variable
for var in config.NUMERICAL_TO_IMPUTE:
X_train = pf.add_missing_indicator(X_train, var)
X_train[var] = pf.impute_na(X_train, var, config.IMPUTATION_DICT[var])
# Group rare labels
for var in config.CATEGORICAL_VARS:
X_train[var] = pf.remove_rare_labels(X_train, var,
config.FREQUENT_LABELS[var])
import preprocessing_functions as pf
import config
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
data = pf.load_data(config.PATH_TO_DATASET)
# divide data set
X_train, X_test, y_train, y_test = pf.divide_train_test(data, config.TARGET)
# get first letter from cabin variable
X_train['cabin'] = pf.extract_cabin_letter(df=X_train, var='cabin')
X_test['cabin'] = pf.extract_cabin_letter(df=X_test, var='cabin')
# impute categorical variables
for var in config.CATEGORICAL_VARS:
X_train[var] = pf.impute_na(X_train, var, replacement='Missing')
X_test[var] = pf.impute_na(X_test, var, replacement='Missing')
# impute numerical variable
for var in config.NUMERICAL_TO_IMPUTE:
X_train[var] = pf.add_missing_indicator(df=X_train, var=var)
X_test[var] = pf.add_missing_indicator(df=X_test, var=var)
# Group rare labels
for var in config.CATEGORICAL_VARS:
X_train[var] = pf.remove_rare_labels(X_train, var,
config.FREQUENT_LABELS[var])
X_test[var] = pf.remove_rare_labels(X_test, var,
import preprocessing_functions as pf
import config
import pandas as pd
# ================================================
# TRAINING STEP - IMPORTANT TO PERPETUATE THE MODEL
# Load data
df = pf.load_data(config.PATH_TO_DATASET)
# divide data set
X_train, X_test, y_train, y_test = pf.divide_train_test(df, config.TARGET)
# get first letter from cabin variable
X_train = pf.extract_cabin_letter(X_train, 'cabin')
X_test = pf.extract_cabin_letter(X_test, 'cabin')
# impute categorical variables
X_train = pf.impute_na(X_train, config.CATEGORICAL_VARS)
X_test = pf.impute_na(X_test, config.CATEGORICAL_VARS)
# impute numerical variable
for var in config.IMPUTATION_DICT.keys():
X_train = pf.add_missing_indicator(X_train, var)
X_test = pf.add_missing_indicator(X_test, var)
X_train = pf.impute_na(X_train, var, config.IMPUTATION_DICT[var])
X_test = pf.impute_na(X_test, var, config.IMPUTATION_DICT[var])
# Group rare labels
for var in config.FREQUENT_LABELS.keys():