def upper_region_classifier():
ur_train_set = pd.read_csv("../resources/datasets/ur_train_set.csv",
na_values='?',
dtype='category')
ur_test_set = pd.read_csv("../resources/datasets/ur_test_set.csv",
na_values='?',
dtype='category')
# Separate training feature & training labels
# X = ur_dataset.drop(['class'], axis=1)
# y = ur_dataset['class']
X_train = ur_train_set.drop(['class'], axis=1)
y_train = ur_train_set['class']
#
# # Separate testing feature & testing labels
X_test = ur_test_set.drop(['class'], axis=1)
y_test = ur_test_set['class']
# get_baseline_performance(X_train, y_train, X_test, y_test)
#
# spot_check_algorithms(X_train, y_train)
model = RandomForestClassifier()
model = model.fit(X_train, y_train)
predictions = model.predict(X_train)
print_evaluation_results(y_train, predictions)
#
predictions = model.predict(X_test)
print_evaluation_results(y_test, predictions, train=False)
joblib.dump(model, filename='../resources/models/ur_classifier.pkl')
def thoracic_region_classifier():
tr_train_set = pd.read_csv("../resources/datasets/tr_train_set.csv",
na_values='?',
dtype='category')
tr_test_set = pd.read_csv("../resources/datasets/tr_test_set.csv",
na_values='?',
dtype='category')
# Separate training feature & training labels
X_train = tr_train_set.drop(['class'], axis=1)
y_train = tr_train_set['class']
# Separate testing feature & testing labels
X_test = tr_test_set.drop(['class'], axis=1)
y_test = tr_test_set['class']
get_baseline_performance(X_train, y_train, X_test, y_test)
model = RandomForestClassifier()
model = model.fit(X_train, y_train)
predictions = model.predict(X_train)
print_evaluation_results(y_train, predictions)
predictions = model.predict(X_test)
print_evaluation_results(y_test, predictions, train=False)
joblib.dump(model, filename='../resources/models/tr_classifier.pkl')
def ep_region_classifier():
ep_dataset = pd.read_csv("../resources/datasets/ep_train_set.csv", na_values='?', dtype='category')
# ep_test_set = pd.read_csv("../resources/datasets/ep_test_set.csv", na_values='?', dtype='category')
# Separate training feature & training labels
X_train = ep_dataset.drop(['class'], axis=1)
y_train = ep_dataset['class']
# Separate testing feature & testing labels
X_test = ep_dataset.drop(['class'], axis=1)
y_test = ep_dataset['class']
X_train, X_Val, y_train, y_val = train_test_split(X_train, y_train, stratify=y_train, test_size=0.1, random_state=42, shuffle=True)
model = RandomForestClassifier()
########################################### Hyper-parameter Tuning ##########################################
# Perform grid search on the classifier using f1 score as the scoring method
grid_obj = GridSearchCV(
estimator=model,
param_grid={
'n_estimators': [10, 20, 30],
'max_depth': [6, 10, 20, 30],
'min_samples_split': [1, 10, 100]
},
n_jobs=-1,
scoring="f1_micro",
cv=5,
verbose=3
)
# Fit the grid search object to the training data and find the optimal parameters
grid_fit = grid_obj.fit(X_train, y_train)
# Get the best estimator
best_clf = grid_fit.best_estimator_
print(best_clf)
predictions = best_clf.predict(X_Val)
print_evaluation_results(y_val, predictions, train=False)
model = best_clf
########################################### Final Model ###########################################
model = model.fit(X_train, y_train)
predictions = model.predict(X_train)
print_evaluation_results(y_train, predictions)
predictions = model.predict(X_test)
print_evaluation_results(y_test, predictions, train=False)
joblib.dump(model, filename='../resources/models/ep_classifier.pkl', protocol=2)
def classify_by_region(data_frame):
get_details(data_frame)
print("Before Oversampling By Region\n", data_frame.groupby('region').size())
# sns.countplot(data_frame['region'], label="Count")
# plt.show()
# sns.heatmap(data_frame.drop('region', axis=1), cmap='cool', annot=True)
# plt.show()
# get_feature_correlations(data_frame, plot=True, return_resulst=False)
X = data_frame.drop(['region', 'class'], axis=1) # Features - drop class from features - 'age', 'sex',
y = data_frame['region'] # Labels
mutual_info = mutual_info_classif(X, y, discrete_features='auto')
print("mutual_info: ", mutual_info)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, shuffle=True)
# X_validation, X_test, y_validation, y_test = train_test_split(X_test, y_test, test_size=0.5, random_state=42, shuffle=True)
sm = BorderlineSMOTE()
X_resampled, y_resampled = sm.fit_sample(X_train, y_train)
print("After Oversampling By Region\n", (pd.DataFrame(y_resampled)).groupby('region').size())
# X_resampled.to_csv('resources/data/X_resampled.csv', index=False)
# y_resampled.to_csv('resources/data/y_resampled.csv', header=['region'], index=False)
###############################################################################
# 4. Scale data #
###############################################################################
# sc = StandardScaler()
# X_resampled = sc.fit_transform(X_resampled)
# X_test = sc.transform(X_test)
# https://datascienceplus.com/selecting-categorical-features-in-customer-attrition-prediction-using-python/
# categorical feature selection
# sf = SelectKBest(chi2, k='all')
# sf_fit = sf.fit(X_train, y_train)
# # print feature scores
# for i in range(len(sf_fit.scores_)):
# print(' %s: %f' % (X_train.columns[i], sf_fit.scores_[i]))
#
# # plot the scores
# datset = pd.DataFrame()
# datset['feature'] = X_train.columns[range(len(sf_fit.scores_))]
# datset['scores'] = sf_fit.scores_
# datset = datset.sort_values(by='scores', ascending=True)
# sns.barplot(datset['scores'], datset['feature'], color='blue')
# sns.set_style('whitegrid')
# plt.ylabel('Categorical Feature', fontsize=18)
# plt.xlabel('Score', fontsize=18)
# # plt.show()
#
sel_chi2 = SelectKBest(chi2, k='all') # chi 10 - 0.64, 0.63, 0.60
X_train_chi2 = sel_chi2.fit_transform(X_resampled, y_resampled)
X_test_chi2 = sel_chi2.transform(X_test)
# Spot Check Algorithms
# spot_check_algorithms(X_resampled, y_resampled)
# models = [SVC(kernel='poly'), RandomForestClassifier(), GradientBoostingClassifier()]
# for i in range(len(models)):
# # Get the final model
# parent_model = models[i] # LR(multiclass-ovr) -0.66, 0.67, 0.67, 0.69, 0.69, 0.68 MLP wid fs - 0.65, 0.69, 0.70, GB - 0.67, without fs 0.62, 0.61, DT - 0.58, RF - 0.67, multi_LR - wid fs 0.64 , voting - 0.60
#
# # Train the final model
# parent_model.fit(X_resampled, y_resampled)
#
# # Evaluate the final model on the training set
# predictions = parent_model.predict(X_resampled)
# print_evaluation_results(y_resampled, predictions)
#
# # Evaluate the final model on the test set
# predictions = parent_model.predict(X_test)
# print_evaluation_results(y_test, predictions, train=False)
# mlp = OneVsRestClassifier(MLPClassifier(hidden_layer_sizes = [100]*5, random_state=42))
pipeline = Pipeline(
[
# ('selector', SelectKBest(f_classif)),
('model', RandomForestClassifier(n_jobs = -1) )
]
)
# Perform grid search on the classifier using f1 score as the scoring method
grid_obj = GridSearchCV(
estimator= GradientBoostingClassifier(),
param_grid={
# 'selector__k': [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17],
'n_estimators': [10, 20, 30],
'max_depth': [6, 10, 20, 30],
# 'max_depth': [1, 10, 20, 30],
'min_samples_split': [1, 10, 100]
# 'model__n_estimators': np.arange(10, 200, 10)
# 'C': [1, 10, 100]
},
n_jobs=-1,
scoring="f1_micro",
cv=5,
verbose=3
)
# Fit the grid search object to the training data and find the optimal parameters
grid_fit = grid_obj.fit(X_resampled, y_resampled)
# Get the best estimator
best_clf = grid_fit.best_estimator_
print(best_clf)
# Get the final model
parent_model = best_clf # LR(multiclass-ovr) -0.66, 0.67, 0.67, 0.69, 0.69, 0.68 MLP wid fs - 0.65, 0.69, 0.70, GB - 0.67, without fs 0.62, 0.61, DT - 0.58, RF - 0.67, multi_LR - wid fs 0.64 , voting - 0.60
t0 = time()
# Train the final model
parent_model.fit(X_resampled, y_resampled)
print("training time:", round(time() - t0, 3), "s")
# Evaluate the final model on the training set
train_predictions = parent_model.predict(X_resampled)
print_evaluation_results(y_resampled, train_predictions)
t0 = time()
# Evaluate the final model on the test set
test_predictions = parent_model.predict(X_test)
print("predicting time:", round(time() - t0, 3), "s")
print_evaluation_results(y_test, test_predictions, train=False)
confusion_matrix(parent_model, X_resampled, y_resampled, X_test, y_test)
def thoracic_region_classifier():
data_frame = pd.read_csv("../resources/datasets/thoracic_region.csv",
na_values='?',
dtype='category')
data_frame.drop('region', axis=1, inplace=True)
get_details(data_frame)
# make_boolean(data_frame)
print("Before Oversampling By Class\n", data_frame.groupby('class').size())
# sns.countplot(data_frame['class'], label="Count")
# plt.show()
features = data_frame.drop(['class'], axis=1)
labels = data_frame['class'] # Labels - 1, 22
X_train, X_test, y_train, y_test = train_test_split(features,
labels,
test_size=0.3,
random_state=42,
shuffle=True)
# pd.DataFrame(X_train).to_csv('resources/data/X_train2.csv', index=False)
# pd.DataFrame(X_test).to_csv('resources/data/X_test2.csv', index=False)
# pd.DataFrame(y_train).to_csv('resources/data/y_train2.csv', index=False)
# pd.DataFrame(y_test).to_csv('resources/data/y_test2.csv', index=False)
print("X_train2 ", pd.DataFrame(X_train).shape)
print("X_train2 ", pd.DataFrame(y_train).shape)
# smote = BorderlineSMOTE()
smote = RandomOverSampler(
) # minority - hamming loss increases, accuracy, jaccard, avg f1, macro avg decreases
X_resampled2, y_resampled2 = smote.fit_sample(X_train, y_train)
# pd.DataFrame(X_resampled2).to_csv('resources/data/X_resampled2.csv', index=False)
# pd.DataFrame(y_resampled2).to_csv('resources/data/y_resampled2.csv', index=False)
print("X_train2 ", pd.DataFrame(X_resampled2).shape)
print("X_train2 ", pd.DataFrame(y_resampled2).shape)
df = pd.DataFrame(y_resampled2)
print(df.groupby('class').size())
# sel_chi2 = SelectKBest(chi2, k=8) # select 8 features
# X_train_chi2 = sel_chi2.fit_transform(X_resampled2, y_resampled2)
# print(sel_chi2.get_support())
#
# X_test_chi2 = sel_chi2.transform(X_test)
# print(X_test.shape)
# print(X_test_chi2.shape)
# # ###############################################################################
# # # 4. Scale data #
# # ###############################################################################
# sc = StandardScaler()
# X_resampled2 = sc.fit_transform(X_resampled2)
# X_test = sc.transform(X_test)
# Spot Check Algorithms
# spot_check_algorithms(X_train_chi2, y_resampled2)
# Make predictions on validation dataset using the selected model
thoracic_model = DecisionTreeClassifier(
) # MLP- 0.88, ExtraTreeClassifier-0.73,0.97,0.94,0.91,0.94, 0.94, 0.86 RF- 0.88, 0.88 GB- 0.89, 0.89 LR()- 0.88, 0.88 LogisticRegression(solver='liblinear', multi_class='ovr') - 0.92, kNN- 0.87, 0.92, 0.84 DT- 0.94, 0.94, 0.89, 0.94 SVC(gamma='auto') - 0.94, MultinomialNB() - 0.88
# models2 = VotingClassifier(
# estimators=[('rf', random_forest), ('knn', KNeighborsClassifier(n_neighbors=5)), ('NB', GaussianNB())],
# voting='hard') # 0.74
# Train the final model
thoracic_model = thoracic_model.fit(X_resampled2, y_resampled2)
# Evaluate the final model on the training set
predictions = thoracic_model.predict(X_resampled2)
print_evaluation_results(y_resampled2, predictions)
# Evaluate the final model on the test set
predictions = thoracic_model.predict(X_test)
print_evaluation_results(y_test, predictions, train=False)
joblib.dump(thoracic_model,
filename='../resources/models/sub_classifier_2.pkl')
def classify_by_region(data_frame):
X = data_frame.drop([TOP_LEVEL_TARGET, SECOND_LEVEL_TARGET],
axis=1) # Features - drop region, class
y = data_frame[TOP_LEVEL_TARGET] # Labels
# ['age', 'degree-of-diffe', 'sex_2', 'histologic-type_2', 'bone_2',
# 'neck_2', 'mediastinum_2', 'abdominal_2']
# data_frame.drop(['lung_2', 'pleura_2', 'peritoneum_2', 'liver_2', 'brain_2', 'skin_2', 'supraclavicular_2',
# 'axillar_2', 'bone-marrow_2'], axis=1, inplace=True)
# get_feature_correlations(data_frame, plot=True, return_resulst=False)
mutual_info = mutual_info_classif(X, y, discrete_features='auto')
print("mutual_info: ", mutual_info)
# 0.3 test size = 0.56 f1
# 0.2 test size = 0.61 f1
X_train, X_test, y_train, y_test = train_test_split(
X,
y,
stratify=y,
test_size=0.2,
random_state=RANDOM_STATE,
shuffle=True)
# reject_sampler = FunctionSampler(func=outlier_rejection)
# X_train, y_train = reject_sampler.fit_resample(X_train, y_train)
# Baseline
# Spot Check Algorithms
# spot_check_algorithms(X_resampled, y_resampled)
########## Handle Class Imabalnce #########
sm = BorderlineSMOTE(random_state=42)
X_resampled, y_resampled = sm.fit_sample(X_train, y_train)
print("After Oversampling By Region\n",
(pd.DataFrame(y_resampled)).groupby('region').size())
# https://datascienceplus.com/selecting-categorical-features-in-customer-attrition-prediction-using-python/
# categorical feature selection
sf = SelectKBest(f_classif, k='all')
sf_fit = sf.fit(X_resampled, y_resampled)
# print feature scores
for i in range(len(sf_fit.scores_)):
print(' %s: %f' % (X_resampled.columns[i], sf_fit.scores_[i]))
# plot the scores
# datset = pd.DataFrame()
# datset['feature'] = X_train.columns[range(len(sf_fit.scores_))]
# datset['scores'] = sf_fit.scores_
# datset = datset.sort_values(by='scores', ascending=True)
# sns.barplot(datset['scores'], datset['feature'], color='blue')
# sns.set_style('whitegrid')
# plt.ylabel('Categorical Feature', fontsize=18)
# plt.xlabel('Score', fontsize=18)
# plt.show()
# models = [SVC(kernel='poly'), RandomForestClassifier(), GradientBoostingClassifier()]
# for i in range(len(models)):
# # Get the final model
# parent_model = models[i] # LR(multiclass-ovr) -0.66, 0.67, 0.67, 0.69, 0.69, 0.68 MLP wid fs - 0.65, 0.69, 0.70, GB - 0.67, without fs 0.62, 0.61, DT - 0.58, RF - 0.67, multi_LR - wid fs 0.64 , voting - 0.60
#
# # Train the final model
# parent_model.fit(X_resampled, y_resampled)
#
# # Evaluate the final model on the training set
# predictions = parent_model.predict(X_resampled)
# print_evaluation_results(y_resampled, predictions)
#
# # Evaluate the final model on the test set
# predictions = parent_model.predict(X_test)
# print_evaluation_results(y_test, predictions, train=False)
model = RandomForestClassifier(RANDOM_STATE)
########################################### Hyper-parameter Tuning ##########################################
best_clf_rf = tune_random_forest(model, X_resampled, y_resampled)
# g = GridSearchCV(
# estimator=GradientBoostingClassifier(),
# param_grid={
# "learning_rate" : [0.05, 0.10, 0.15, 0.20, 0.25, 0.30 ] ,
# "max_depth" : [ 3, 4, 5, 6, 8, 10, 12, 15],
# "min_child_weight" : [ 1, 3, 5, 7 ],
# "gamma" : [ 0.0, 0.1, 0.2 , 0.3, 0.4 ],
# "colsample_bytree" : [ 0.3, 0.4, 0.5 , 0.7 ]
# },
# n_jobs=-1,
# scoring="f1_micro",
# cv=5,
# verbose=1
# )
# # Fit the grid search object to the training data and find the optimal parameters
# grid_fit = grid_obj.fit(X_resampled, y_resampled)
#
# # Get the best estimator
# best_clf_gb= grid_fit.best_estimator_
# print(best_clf_gb)
########################################### Final Model ###########################################
parent_model = best_clf_rf # LR(multiclass-ovr) -0.66, 0.67, 0.67, 0.69, 0.69, 0.68 MLP wid fs - 0.65, 0.69, 0.70, GB - 0.67, without fs 0.62, 0.61, DT - 0.58, RF - 0.67, multi_LR - wid fs 0.64 , voting - 0.60
t0 = time()
# Train the final model
parent_model.fit(X_resampled, y_resampled)
print("training time:", round(time() - t0, 3), "s")
# Evaluate the final model on the training set
train_predictions = parent_model.predict(X_resampled)
print_evaluation_results(y_resampled, train_predictions)
t0 = time()
# Evaluate the final model on the test set
test_predictions = parent_model.predict(X_test)
print("predicting time:", round(time() - t0, 3), "s")
print_evaluation_results(y_test, test_predictions, train=False)
confusion_matrix(parent_model, X_resampled, y_resampled, X_test, y_test)
# https://towardsdatascience.com/feature-selection-using-random-forest-26d7b747597f
sel = SelectFromModel(best_clf_rf)
sel.fit(X_resampled, y_resampled)
print(sel.get_support())
selected_feat = X_resampled.columns[(sel.get_support())]
print(len(selected_feat))
print(selected_feat)
def ip_region_classifier():
ip_train_set = pd.read_csv("../resources/datasets/ip_train_set.csv",
dtype='category')
ip_test_set = pd.read_csv("../resources/datasets/ip_test_set.csv",
dtype='category')
# print("ip missing ", ip_train_set.isnull().sum().sum())
# get_feature_correlations(ip_train_set)
# Separate training feature & training labels
X_train = ip_train_set.drop(['class'], axis=1)
y_train = ip_train_set['class']
# Separate testing feature & testing labels
X_test = ip_test_set.drop(['class'], axis=1)
y_test = ip_test_set['class']
get_baseline_performance(X_train, y_train, X_test, y_test)
model = RandomForestClassifier(random_state=RANDOM_STATE)
model = model.fit(X_train, y_train)
# https://towardsdatascience.com/machine-learning-kaggle-competition-part-two-improving-e5b4d61ab4b8
# https://www.kaggle.com/residentmario/automated-feature-selection-with-sklearn
# pd.Series(model.feature_importances_, index=X_train.columns[0:]).plot.bar(color='steelblue', figsize=(12, 6))
# plt.show()
# from sklearn.feature_selection import mutual_info_classif
# kepler_mutual_information = mutual_info_classif(X_train, y_train)
# plt.subplots(1, figsize=(26, 1))
# sns.heatmap(kepler_mutual_information[:, np.newaxis].T, cmap='Blues', cbar=False, linewidths=1, annot=True)
# plt.yticks([], [])
# plt.gca().set_xticklabels(X_train.columns[0:], rotation=45, ha='right', fontsize=12)
# plt.suptitle("Kepler Variable Importance (mutual_info_classif)", fontsize=18, y=1.2)
# plt.gcf().subplots_adjust(wspace=0.2)
# plt.show()
#
# trans = GenericUnivariateSelect(score_func=mutual_info_classif, mode='percentile', param=50)
# kepler_X_trans = trans.fit_transform(X_train, y_train)
# kepler_X_test_trans = trans.transform(X_test)
# print("We started with {0} features but retained only {1} of them!".format(X_train.shape[1] - 1,
# kepler_X_trans.shape[1]))
# https://www.kaggle.com/yaldazare/feature-selection-and-data-visualization
# we will not only find best features but we also find how many features do we need for best accuracy.
# The "accuracy" scoring is proportional to the number of correct classifications
clf_rf_4 = RandomForestClassifier()
# cv = KFold(n_repeats=3, n_splits=10, random_state=42)
rfecv = RFECV(estimator=clf_rf_4, step=1, cv=5,
scoring='f1_micro') # 5-fold cross-validation
rfecv = rfecv.fit(X_train, y_train)
print('Optimal number of features :', rfecv.n_features_)
print('Best features :', X_train.columns[rfecv.support_])
# Plot number of features VS. cross-validation scores
import matplotlib.pyplot as plt
plt.figure()
plt.xlabel("Number of features selected")
plt.ylabel("Cross validation score of number of selected features")
plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)
plt.show()
clr_rf_5 = model.fit(X_train, y_train)
importances = clr_rf_5.feature_importances_
std = np.std([tree.feature_importances_ for tree in model.estimators_],
axis=0)
indices = np.argsort(importances)[::-1]
# Print the feature ranking
print("Feature ranking:")
for f in range(X_train.shape[1]):
print("%d. feature %d (%f)" %
(f + 1, indices[f], importances[indices[f]]))
# Plot the feature importances of the forest
plt.figure(1, figsize=(14, 13))
plt.title("Feature importances")
plt.bar(range(X_train.shape[1]),
importances[indices],
color="g",
yerr=std[indices],
align="center")
plt.xticks(range(X_train.shape[1]), X_train.columns[indices], rotation=90)
plt.xlim([-1, X_train.shape[1]])
plt.show()
predictions = model.predict(X_train)
print_evaluation_results(y_train, predictions)
predictions = model.predict(X_test)
print_evaluation_results(y_test, predictions, train=False)
joblib.dump(model, filename='../resources/models/ip_classifier.pkl')
def upper_region_classifier():
ur_train_set = pd.read_csv("../resources/datasets/ur_train_set.csv",
na_values='?',
dtype='category')
ur_test_set = pd.read_csv("../resources/datasets/ur_test_set.csv",
na_values='?',
dtype='category')
# Separate training feature & training labels
X = ur_train_set.drop([SECOND_LEVEL_TARGET], axis=1)
y = ur_train_set[SECOND_LEVEL_TARGET]
# Separate testing feature & testing labels
X_test = ur_test_set.drop([SECOND_LEVEL_TARGET], axis=1)
y_test = ur_test_set[SECOND_LEVEL_TARGET]
# Dividing training set into sub-training & validation set
X_train, X_val, y_train, y_val = train_test_split(X,
y,
test_size=0.1,
stratify=y)
# get_baseline_performance(X_train, y_train, X_test, y_test)
# spot_check_algorithms(X_train, y_train)
model = RandomForestClassifier()
model = model.fit(X_train, y_train)
predictions = model.predict(X_train)
print_evaluation_results(y_train, predictions)
predictions = model.predict(X_test)
print_evaluation_results(y_test, predictions, train=False)
svm_model = SVC(kernel='poly', gamma=0.1, C=1.0)
############################################# Hyper-parameter Tuning #######################################£££####
# cv = RepeatedStratifiedKFold(n_splits=3, random_state=42)
params = {
'n_estimators': [5, 10, 20, 30],
'max_depth': [4, 6, 10, 12],
'random_state': [13]
}
# Instantiate the grid search model
grid_search = GridSearchCV(estimator=model,
param_grid=params,
scoring="accuracy",
cv=5,
n_jobs=-1,
verbose=1)
grid_search.fit(X_train, y_train)
print(grid_search.best_params_)
print(grid_search.best_score_)
model = grid_search.best_estimator_
params = {
"C": (0.1, 0.5, 1, 2, 5, 10, 20),
"gamma": (0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 1),
"kernel": ('linear', 'poly', 'rbf')
}
svm_grid = GridSearchCV(svm_model,
params,
n_jobs=-1,
cv=5,
verbose=1,
scoring="accuracy")
svm_grid.fit(X_train, y_train)
print(svm_grid.best_params_)
print(svm_grid.best_score_)
n_estimators = [int(x) for x in np.linspace(start=200, stop=2000, num=10)]
max_features = ['auto', 'sqrt']
max_depth = [int(x) for x in np.linspace(10, 110, num=11)]
max_depth.append(None)
min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
bootstrap = [True, False]
random_grid = {
'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap
}
rand_forest = RandomForestClassifier(random_state=42)
rf_random = RandomizedSearchCV(estimator=rand_forest,
param_distributions=random_grid,
n_iter=100,
cv=3,
verbose=2,
random_state=42,
n_jobs=-1)
rf_random.fit(X_train, y_train)
print(rf_random.best_params_)
print(rf_random.best_score_)
model = tune_random_forest(model, X_train, y_train)
# model = rf_random.best_estimator_
################################################## Model Evaluation ###########################################£££####
predictions = model.predict(X_train)
print_evaluation_results(y_train, predictions)
predictions = model.predict(X_test)
print_evaluation_results(y_test, predictions, train=False)
def classify_by_region(data_frame):
X = data_frame.drop([TOP_LEVEL_TARGET, SECOND_LEVEL_TARGET], axis=1) # Features - drop region, class
y = data_frame[TOP_LEVEL_TARGET] # Labels
# get_feature_correlations(data_frame, plot=True, return_resulst=False)
# mutual_info = mutual_info_classif(X, y, discrete_features='auto')
# print("mutual_info: ", mutual_info)
X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size=0.2, random_state=42, shuffle=True)
########## Handle Class Imabalnce #########
sm = BorderlineSMOTE()
X_resampled, y_resampled = sm.fit_sample(X_train, y_train)
print("After Oversampling By Region\n", (pd.DataFrame(y_resampled)).groupby('region').size())
###############################################################################
# 4. Scale data #
###############################################################################
# sc = StandardScaler()
# X_resampled = sc.fit_transform(X_resampled)
# X_test = sc.transform(X_test)
# https://datascienceplus.com/selecting-categorical-features-in-customer-attrition-prediction-using-python/
# categorical feature selection
# sf = SelectKBest(chi2, k='all')
# sf_fit = sf.fit(X_train, y_train)
# # print feature scores
# for i in range(len(sf_fit.scores_)):
# print(' %s: %f' % (X_train.columns[i], sf_fit.scores_[i]))
#
# # plot the scores
# datset = pd.DataFrame()
# datset['feature'] = X_train.columns[range(len(sf_fit.scores_))]
# datset['scores'] = sf_fit.scores_
# datset = datset.sort_values(by='scores', ascending=True)
# sns.barplot(datset['scores'], datset['feature'], color='blue')
# sns.set_style('whitegrid')
# plt.ylabel('Categorical Feature', fontsize=18)
# plt.xlabel('Score', fontsize=18)
# # plt.show()
#
sel_chi2 = SelectKBest(chi2, k='all') # chi 10 - 0.64, 0.63, 0.60
X_train_chi2 = sel_chi2.fit_transform(X_resampled, y_resampled)
X_test_chi2 = sel_chi2.transform(X_test)
# mlp = OneVsRestClassifier(MLPClassifier(hidden_layer_sizes = [100]*5, random_state=42))
# Spot Check Algorithms
# spot_check_algorithms(X_resampled, y_resampled)
# models = [SVC(kernel='poly'), RandomForestClassifier(), GradientBoostingClassifier()]
# for i in range(len(models)):
# # Get the final model
# parent_model = models[i] # LR(multiclass-ovr) -0.66, 0.67, 0.67, 0.69, 0.69, 0.68 MLP wid fs - 0.65, 0.69, 0.70, GB - 0.67, without fs 0.62, 0.61, DT - 0.58, RF - 0.67, multi_LR - wid fs 0.64 , voting - 0.60
#
# # Train the final model
# parent_model.fit(X_resampled, y_resampled)
#
# # Evaluate the final model on the training set
# predictions = parent_model.predict(X_resampled)
# print_evaluation_results(y_resampled, predictions)
#
# # Evaluate the final model on the test set
# predictions = parent_model.predict(X_test)
# print_evaluation_results(y_test, predictions, train=False)
# pipeline = Pipeline(
# [
# # ('selector', SelectKBest(f_classif)),
# ('model', RandomForestClassifier(n_jobs = -1) )
# ]
# )
#
# # Perform grid search on the classifier using f1 score as the scoring method
# grid_obj = GridSearchCV(
# estimator= GradientBoostingClassifier(),
# param_grid={
# # 'selector__k': [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17],
# 'n_estimators': [10, 20, 30],
# 'max_depth': [6, 10, 20, 30],
# # 'max_depth': [1, 10, 20, 30],
# 'min_samples_split': [1, 10, 100]
# # 'model__n_estimators': np.arange(10, 200, 10)
# # 'C': [1, 10, 100]
# },
#
# n_jobs=-1,
# scoring="f1_micro",
# cv=5,
# verbose=3
# )
#
# # Fit the grid search object to the training data and find the optimal parameters
# grid_fit = grid_obj.fit(X_resampled, y_resampled)
# # Get the best estimator
# best_clf = grid_fit.best_estimator_
# print(best_clf)
# Get the final model
parent_model = SVC(kernel = 'rbf', C = 10)#KNN(n_neighbors = 7)-0.52 # LR(multiclass-ovr) -0.66, 0.67, 0.67, 0.69, 0.69, 0.68 MLP wid fs - 0.65, 0.69, 0.70, GB - 0.67, without fs 0.62, 0.61, DT - 0.58, RF - 0.67, multi_LR - wid fs 0.64 , voting - 0.60
t0 = time()
# Train the final model
parent_model.fit(X_resampled, y_resampled)
print("training time:", round(time() - t0, 3), "s")
# Evaluate the final model on the training set
train_predictions = parent_model.predict(X_resampled)
print_evaluation_results(y_resampled, train_predictions)
t0 = time()
# Evaluate the final model on the test set
test_predictions = parent_model.predict(X_test)
print("predicting time:", round(time() - t0, 3), "s")
print_evaluation_results(y_test, test_predictions, train=False)
confusion_matrix(parent_model, X_resampled, y_resampled, X_test, y_test)
# Plot normalized confusion matrix
# fig = plt.figure()
# fig.set_size_inches(8, 8, forward=True)
# # fig.align_labels()
# plot_confusion_matrix(cnf_matrix, classes=["1", "2", "3", "4"], normalize=False, title='Normalized confusion matrix')
# probs = parent_model.predict_proba(X_test)
# print("Prediction probabilities for Region\n", probs)
# plotConfusionMatrix(X_test, y_test, ['1', '2', '3', '4'])
joblib.dump(parent_model, filename='../resources/models/parent_classifier.pkl')
def ip_region_classifier():
ip_dataset = pd.read_csv("../resources/datasets/ip_dataset.csv", dtype='category')
#
# # Separate training feature & training labels
X = ip_dataset.drop([SECOND_LEVEL_TARGET], axis=1)
y = ip_dataset[SECOND_LEVEL_TARGET]
#
# # Spot check
# # spot_check_algorithms(X, y)
#
# # Create a cross-validation strategy
cv = RepeatedStratifiedKFold(n_splits=3, random_state=42)
imba_pipeline = make_pipeline(RandomOverSampler(random_state=42),
SelectKBest(k=17),
RandomForestClassifier(n_estimators=500, max_depth=2, random_state=42))
scoring = ['accuracy', 'f1_micro', 'precision_micro', 'recall_micro']
cv_results = cross_validate(imba_pipeline, X, y, cv=cv, scoring=scoring, n_jobs=-1)
print(sorted(cv_results.keys()))
print(cv_results['fit_time'].mean())
print(cv_results['score_time'].mean())
print(cv_results['test_accuracy'].mean())
print(cv_results['test_f1_micro'].mean())
print(cv_results['test_precision_micro'].mean())
print(cv_results['test_recall_micro'].mean())
# Separate training feature & training labels
X = ip_dataset.drop([SECOND_LEVEL_TARGET], axis=1)
y = ip_dataset[SECOND_LEVEL_TARGET]
# Separate testing feature & testing labels
X_test = ip_dataset.drop([SECOND_LEVEL_TARGET], axis=1)
y_test = ip_dataset[SECOND_LEVEL_TARGET]
get_baseline_performance(X, y, X_test, y_test)
model = RandomForestClassifier(random_state=42)
model = model.fit(X, y)
# https://towardsdatascience.com/machine-learning-kaggle-competition-part-two-improving-e5b4d61ab4b8
# https://www.kaggle.com/residentmario/automated-feature-selection-with-sklearn
# pd.Series(model.feature_importances_, index=X_train.columns[0:]).plot.bar(color='steelblue', figsize=(12, 6))
# plt.show()
# from sklearn.feature_selection import mutual_info_classif
# kepler_mutual_information = mutual_info_classif(X_train, y_train)
# plt.subplots(1, figsize=(26, 1))
# sns.heatmap(kepler_mutual_information[:, np.newaxis].T, cmap='Blues', cbar=False, linewidths=1, annot=True)
# plt.yticks([], [])
# plt.gca().set_xticklabels(X_train.columns[0:], rotation=45, ha='right', fontsize=12)
# plt.suptitle("Kepler Variable Importance (mutual_info_classif)", fontsize=18, y=1.2)
# plt.gcf().subplots_adjust(wspace=0.2)
# plt.show()
#
# trans = GenericUnivariateSelect(score_func=mutual_info_classif, mode='percentile', param=50)
# kepler_X_trans = trans.fit_transform(X_train, y_train)
# kepler_X_test_trans = trans.transform(X_test)
# print("We started with {0} features but retained only {1} of them!".format(X_train.shape[1] - 1,
# kepler_X_trans.shape[1]))
# https://www.kaggle.com/yaldazare/feature-selection-and-data-visualization
# we will not only find best features but we also find how many features do we need for best accuracy.
# The "accuracy" scoring is proportional to the number of correct classifications
clf_rf_4 = RandomForestClassifier()
# cv = KFold(n_repeats=3, n_splits=10, random_state=42)
rfecv = RFECV(estimator=clf_rf_4, step=1, cv=5, scoring='f1_micro') # 5-fold cross-validation
rfecv = rfecv.fit(X, y)
print('Optimal number of features :', rfecv.n_features_)
print('Best features :', X.columns[rfecv.support_])
import matplotlib.pyplot as plt
# Plot number of features VS. cross-validation scores
import matplotlib.pyplot as plt
plt.figure()
plt.xlabel("Number of features selected")
plt.ylabel("Cross validation score of number of selected features")
plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)
plt.show()
clr_rf_5 = model.fit(X, y)
importances = clr_rf_5.feature_importances_
std = np.std([tree.feature_importances_ for tree in model.estimators_], axis=0)
indices = np.argsort(importances)[::-1]
# Print the feature ranking
print("Feature ranking:")
for f in range(X.shape[1]):
print("%d. feature %d (%f)" % (f + 1, indices[f], importances[indices[f]]))
# Plot the feature importances of the forest
plt.figure(1, figsize=(14, 13))
plt.title("Feature importances")
plt.bar(range(X.shape[1]), importances[indices], color="g", yerr=std[indices], align="center")
plt.xticks(range(X.shape[1]), X.columns[indices], rotation=90)
plt.xlim([-1, X.shape[1]])
plt.show()
predictions = model.predict(X)
print_evaluation_results(y, predictions)
predictions = model.predict(X_test)
print_evaluation_results(y_test, predictions, train=False)
# joblib.dump(model, filename='../resources/models/ip_classifier.pkl')
print("Intraperitoneal region Columns ", list(X.columns))
final_model = model.fit(X, y)
joblib.dump(final_model, filename='../resources/models/IPRModel.pkl')
def extra_peritoneum_region_classifier():
data_frame = pd.read_csv(
"../resources/datasets/extra_peritoneum_region.csv",
na_values='?',
dtype='category')
data_frame.drop('region', axis=1, inplace=True)
get_details(data_frame)
print("Before Oversampling By Class\n", data_frame.groupby('class').size())
# make_boolean(data_frame)
# sns.countplot(data_frame['class'], label="Count")
# plt.show()
features = data_frame.drop(['class'], axis=1)
labels = data_frame['class'] # Labels - 8, 14, 15, 16, 17, 18, 19, 20, 21
# pca = decomposition.PCA(n_components=9)
# pca.fit(features)
# features = pca.transform(features)
X_train, X_test, y_train, y_test = train_test_split(features,
labels,
test_size=0.3,
random_state=42,
shuffle=True)
# pd.DataFrame(X_train).to_csv('resources/data/X_train2.csv', index=False)
# pd.DataFrame(X_test).to_csv('resources/data/X_test2.csv', index=False)
# pd.DataFrame(y_train).to_csv('resources/data/y_train2.csv', index=False)
# pd.DataFrame(y_test).to_csv('resources/data/y_test2.csv', index=False)
print("X_train2 ", pd.DataFrame(X_train).shape)
print("X_train2 ", pd.DataFrame(y_train).shape)
# smote = RandomOverSampler() # minority - hamming loss increases, accuracy, jaccard, avg f1, macro avg decreases
# X_resampled2, y_resampled2 = smote.fit_sample(X_train2, y_train2)
# # X_resampled2, y_resampled2 = SMOTE().fit_resample(X_resampled2, y_resampled2)
# pd.DataFrame(X_resampled2).to_csv('resources/data/X_resampled2.csv', index=False)
# pd.DataFrame(y_resampled2).to_csv('resources/data/y_resampled2.csv', index=False)
X_resampled2 = None
y_resampled2 = None
smote = RandomOverSampler()
# for i in range(4):
# X_resampled2, y_resampled2 = smote.fit_resample(X_train2, y_train2)
# X_train2 = X_resampled2
# y_train2 = y_resampled2
# print("X_train2 ", pd.DataFrame(X_resampled2).shape)
# print("X_train2 ", pd.DataFrame(y_resampled2).shape)
X_resampled, y_resampled = smote.fit_resample(X_train, y_train)
# pd.DataFrame(X_resampled).to_csv('resources/data/X_resampled2.csv', index=False)
# pd.DataFrame(y_resampled).to_csv('resources/data/y_resampled2.csv', index=False)
# print("X_train2 ", pd.DataFrame(X_resampled2).shape)
# print("X_train2 ", pd.DataFrame(y_resampled2).shape)
df = pd.DataFrame(y_resampled)
print(df.groupby('class').size())
sel_chi2 = SelectKBest(chi2, k=8) # select 8 features
X_train_chi2 = sel_chi2.fit_transform(X_resampled, y_resampled)
print(sel_chi2.get_support())
X_test_chi2 = sel_chi2.transform(X_test)
print(X_test.shape)
print(X_test_chi2.shape)
# # ###############################################################################
# # # 4. Scale data #
# # ###############################################################################
# sc = StandardScaler()
# X_resampled2 = sc.fit_transform(X_resampled2)
# X_test2 = sc.transform(X_test2)
estimators = [('rf', RandomForestClassifier(random_state=42)),
('svr',
make_pipeline(StandardScaler(), KNeighborsClassifier()))]
# clf = StackingClassifier(estimators=estimators, final_estimator=LogisticRegression(multi_class='ovr'))
clf = StackingClassifier(
estimators=estimators,
final_estimator=LogisticRegression(multi_class='ovr'))
# Spot Check Algorithms
spot_check_algorithms(X_train_chi2, y_resampled)
# Make predictions on validation dataset using the selected model
# model2 = OneVsRestClassifier(GaussianNB()) # 0.43
# model2 = DecisionTreeClassifier() # 0.78, 0.86, 0.72
# model2 = RandomForestClassifier() # 0.80, 0.74, 0.69, 0.65, 0.66
# model2 = GradientBoostingClassifier() #
# model2 = VotingClassifier(estimators=[('rf', RandomForestClassifier()), ('mlp', MLPClassifier()), (('NB', GaussianNB()))], voting='hard') # 0.51
extra_peritoneum_model = OneVsRestClassifier(
RandomForestClassifier()
) # -0.48, wid 8 features - 0.48, 0.52, 0.53, 0.54, 0.57, wid * f - 0.51, clf - 0.48, kNN - 0.49
# Train the final model
extra_peritoneum_model = extra_peritoneum_model.fit(
X_train_chi2, y_resampled)
# Evaluate the final model on the training set
predictions = extra_peritoneum_model.predict(X_train_chi2)
print_evaluation_results(y_resampled, predictions)
# Evaluate the final model on the test set
predictions = extra_peritoneum_model.predict(X_test_chi2)
print_evaluation_results(y_test, predictions, train=False)
joblib.dump(extra_peritoneum_model,
filename='../resources/models/sub_classifier_4.pkl')
def upper_region_classifier():
# Read in data
data_frame = pd.read_csv("../resources/datasets/upper_region.csv",
na_values='?',
dtype='category')
data_frame.drop('region', axis=1, inplace=True)
get_details(data_frame)
print("Before Oversampling By Class\n", data_frame.groupby('class').size())
# sns.countplot(data_frame['class'], label="Count")
# plt.show()
features = data_frame.drop(['class'], axis=1)
labels = data_frame['class'] # Labels - 2, 4, 10
X_train, X_test, y_train, y_test = train_test_split(features,
labels,
test_size=0.3,
random_state=42,
shuffle=True)
print("X_train2 ", pd.DataFrame(X_train).shape)
print("X_train2 ", pd.DataFrame(y_train).shape)
ros = RandomOverSampler(
) # minority - hamming loss increases, accuracy, jaccard, avg f1, macro avg decreases
X_resampled, y_resampled = ros.fit_sample(X_train, y_train)
print("X_train2 ", pd.DataFrame(X_resampled).shape)
print("X_train2 ", pd.DataFrame(y_resampled).shape)
df = pd.DataFrame(y_resampled)
print(df.groupby('class').size())
# https://datascienceplus.com/selecting-categorical-features-in-customer-attrition-prediction-using-python/
# categorical feature selection
# sf = SelectKBest(chi2, k='all')
# sf_fit = sf.fit(X_train, y_train)
# # print feature scores
# for i in range(len(sf_fit.scores_)):
# print(' %s: %f' % (X_train.columns[i], sf_fit.scores_[i]))
#
# # plot the scores
# datset = pd.DataFrame()
# datset['feature'] = X_train.columns[range(len(sf_fit.scores_))]
# datset['scores'] = sf_fit.scores_
# datset = datset.sort_values(by='scores', ascending=True)
# sns.barplot(datset['scores'], datset['feature'], color='blue')
# sns.set_style('whitegrid')
# plt.ylabel('Categorical Feature', fontsize=18)
# plt.xlabel('Score', fontsize=18)
# plt.show()
# sel_chi2 = SelectKBest(chi2, k=9) # DT chi 9- 0.83*2, 10- 91,92
# # RF chi 9- 1,1,91,1,91
# # GB chi 9- 91,91,82,91
# X_train_chi2 = sel_chi2.fit_transform(X_resampled, y_resampled)
# X_test_chi2 = sel_chi2.transform(X_test)
# # ###############################################################################
# # # 4. Scale data #
# # ###############################################################################
# sc = StandardScaler()
# X_resampled2 = sc.fit_transform(X_resampled2)
# X_test2 = sc.transform(X_test2)
# get_baseline_performance(X_resampled, y_resampled, X_test, y_test)
# Spot Check Algorithms
# spot_check_algorithms(X_resampled, y_resampled)
# Make predictions on validation dataset using the selected model
# upper_region_model = MLP # DT()- 0.92*4, RF()-0.91, RF(n_estimators=200) - 0.91, 1.0, # kNN - 0.91, knn(neigh-3) - 0.78, knn(neigh-5) - 0.91, DT - 0.79, 0.91, 0.92, SVC(gamma='auto') - 0.91, LogisticRegression(solver='liblinear', multi_class='ovr') - 0.85,
upper_region_model = RandomForestClassifier(n_jobs=-1,
max_depth=20,
n_estimators=200)
# define Boruta feature selection method
# feat_selector = BorutaPy(upper_region_model, n_estimators='auto', verbose=2, random_state=1)
# # find all relevant features - 5 features should be selected
# feat_selector.fit(X_resampled, y_resampled)
# # check selected features - first 5 features are selected
# print(feat_selector.support_)
# # check ranking of features
# print(feat_selector.ranking_)
# # call transform() on X to filter it down to selected features
# X_filtered = feat_selector.transform(X_resampled)
# Train the final model
upper_region_model = upper_region_model.fit(X_resampled, y_resampled)
# Evaluate the final model on the training set
predictions = upper_region_model.predict(X_resampled)
print_evaluation_results(y_resampled, predictions)
# Evaluate the final model on the test set
predictions = upper_region_model.predict(X_test)
print_evaluation_results(y_test, predictions, train=False)
def intra_peritoneum_region_classifier():
data_frame = pd.read_csv(
"../resources/datasets/intra_peritoneum_region.csv",
na_values='?',
dtype='category')
data_frame.drop('region', axis=1, inplace=True)
get_details(data_frame)
print("Before Oversampling By Class\n", data_frame.groupby('class').size())
# sns.countplot(data_frame['class'], label="Count")
# plt.show()
features = data_frame.drop(['class'], axis=1)
labels = data_frame['class'] # Labels - 3, 5, 6, 7, 11, 12, 13
# pca = decomposition.PCA(n_components=9)
# pca.fit(features)
# features = pca.transform(features)
X_train, X_test, y_train, y_test = train_test_split(features,
labels,
test_size=0.3,
random_state=42,
shuffle=True)
# pd.DataFrame(X_train).to_csv('resources/data/X_train2.csv', index=False)
# pd.DataFrame(X_test).to_csv('resources/data/X_test2.csv', index=False)
# pd.DataFrame(y_train).to_csv('resources/data/y_train2.csv', index=False)
# pd.DataFrame(y_test).to_csv('resources/data/y_test2.csv', index=False)
print("X_train2 ", pd.DataFrame(X_train).shape)
print("X_train2 ", pd.DataFrame(y_train).shape)
smote = RandomOverSampler(
sampling_strategy='minority'
) # minority - hamming loss increases, accuracy, jaccard, avg f1, macro avg decreases
X_resampled, y_resampled = smote.fit_sample(X_train, y_train)
# X_resampled2, y_resampled2 = SMOTE().fit_resample(X_resampled2, y_resampled2)
# pd.DataFrame(X_resampled).to_csv('resources/data/X_resampled2.csv', index=False)
# pd.DataFrame(y_resampled).to_csv('resources/data/y_resampled2.csv', index=False)
print("X_train2 ", pd.DataFrame(X_resampled).shape)
print("X_train2 ", pd.DataFrame(y_resampled).shape)
df = pd.DataFrame(y_resampled)
print(df.groupby('class').size())
sel_chi2 = SelectKBest(chi2, k=8) # select 9 features
X_train_chi2 = sel_chi2.fit_transform(X_resampled, y_resampled)
print(sel_chi2.get_support())
X_test_chi2 = sel_chi2.transform(X_test)
print(X_test.shape)
print(X_test_chi2.shape)
# # ###############################################################################
# # # 4. Scale data #
# # ###############################################################################
# sc = StandardScaler()
# X_resampled2 = sc.fit_transform(X_resampled2)
# X_test2 = sc.transform(X_test2)
# Spot Check Algorithms
# spot_check_algorithms(X_train_chi2, y_resampled)
# Make predictions on validation dataset using the selected model
# intra_peritoneum_model = KNeighborsClassifier(n_neighbors=5) # kNN()- 0.39, kNN(neig-5) - 0.44, 0.39, LogisticRegression(solver='liblinear', multi_class='ovr'), wid 8, 9 features - 0.42, wid 12 featues - 0.40, SVC(gamma='auto') - 0.35, OneVsRestClassifier(GaussianNB()) - 0.05
intra_peritoneum_model = IsolationForest(n_estimators=100)
# Train the final model
intra_peritoneum_model = intra_peritoneum_model.fit(
X_train_chi2, y_resampled)
# Evaluate the final model on the training set
predictions = intra_peritoneum_model.predict(X_train_chi2)
print_evaluation_results(y_resampled, predictions)
# Evaluate the final model on the test set
predictions = intra_peritoneum_model.predict(X_test_chi2)
print_evaluation_results(y_test, predictions, train=False)
joblib.dump(intra_peritoneum_model,
filename='../resources/models/sub_classifier_3.pkl')