class ModelB(object):
def __init__(self, epochs, batch_size, dataset_name, learning_rate,
model_name, is_binarized, is_resized, is_grayscale):
self.dataset_name = dataset_name
self.learning_rate = learning_rate
self.model_name = model_name
self.no_epochs = epochs
self.batch_size = batch_size
self.is_binarized = is_binarized
self.is_resized = is_resized
self.is_grayscale = is_grayscale
if (dataset_name == 'mnist'):
if (self.is_resized):
self.input_image_size = (10, 10, 1)
else:
self.input_image_size = (28, 28, 1)
if (is_binarized):
self.output_classes = 2
else:
self.output_classes = 10
elif (dataset_name == 'fashion_mnist'):
self.input_image_size = (10, 10,
1) if (self.is_resized) else (28, 28, 1)
self.output_classes = 2 if (self.is_binarized) else 10
elif (dataset_name == 'stanford40'):
self.input_image_size = (200, 200, 3)
self.output_classes = 40
else:
print("Not Implemented yet")
if (self.model_name == 'ann'):
input_size = np.prod(self.input_image_size)
self.hidden_layers = hidden_layer_list_model_B
self.NN = Neural_Network(self.no_epochs,
self.batch_size,
self.learning_rate,
input_size,
self.output_classes,
self.hidden_layers,
mc_dropout=False,
dropout_rate=None)
elif (self.model_name == 'svm'):
self.classifier = svm.SVC(C=1,
kernel='rbf',
gamma='auto',
probability=True,
random_state=0)
elif (self.model_name == "naive_bayes"):
self.classifier = MultinomialNB(alpha=1.0)
elif (self.model_name == 'dt'):
self.classifier = DecisionTree('gini', 'best', None, 10)
elif (self.model_name == 'cnn'):
self.CNN_classifier = CNN(self.input_image_size, self.no_epochs,
self.batch_size, self.output_classes,
self.learning_rate)
elif (self.model_name == 'inceptionv3'):
self.inception_classifier = Inceptionv3(self.output_classes,
self.batch_size,
self.no_epochs,
self.input_image_size)
#print("Parameters Initialized")
def set_dataset(self, data_X, data_Y, test_X, test_Y, cross_validation_X,
cross_validation_Y):
self.data_X, self.data_Y, self.test_X, self.test_Y, self.cross_validation_X, self.cross_validation_Y = data_X, data_Y, test_X, test_Y, cross_validation_X, cross_validation_Y
self.no_batches = len(self.data_X) // (self.batch_size)
def init_model(self):
if (self.model_name == 'ann'):
self.NN.create_tf_model("ModelB")
elif (self.model_name == 'cnn'):
self.CNN_classifier.initialize_model()
elif (self.model_name == 'inceptionv3'):
self.inception_classifier.initialize_model()
elif (self.model_name == "svm" or self.model_name == "dt"
or self.model_name == "naive_bayes"):
pass
else:
print("Not implemented yet")
##print("Model Initialized")
def train_model(self):
if (self.model_name == 'cnn'):
self.CNN_classifier.train_model(self.data_X, self.data_Y,
self.cross_validation_X,
self.cross_validation_Y)
elif (self.model_name == 'inceptionv3'):
self.inception_classifier.train_model(self.data_X, self.data_Y,
self.cross_validation_X,
self.cross_validation_Y)
else:
self.data_X = self.data_X.reshape(self.data_X.shape[0], -1)
self.cross_validation_X = self.cross_validation_X.reshape(
self.cross_validation_X.shape[0], -1)
self.test_X = self.test_X.reshape(self.test_X.shape[0], -1)
if (self.model_name == 'svm' or self.model_name == 'naive_bayes'):
classes_ = np.unique(self.data_Y)
for class_ in classes_:
indices = (self.data_Y == class_).nonzero()
print("Class: ", class_, "No of samples: ",
np.array(indices).shape)
classes_ = np.unique(self.test_Y)
for class_ in classes_:
indices = (self.test_Y == class_).nonzero()
print("Class: ", class_, "No of samples: ",
np.array(indices).shape)
self.classifier.fit(self.data_X, self.data_Y)
elif (self.model_name == 'ann'):
self.NN.train_model(self.data_X, self.data_Y,
self.cross_validation_X,
self.cross_validation_Y)
elif (self.model_name == 'dt'):
self.classifier.train_model(self.data_X, self.data_Y,
self.cross_validation_X,
self.cross_validation_Y)
else:
print("Not yet implemented")
def get_output(self, mode='original'):
tr_X, tr_Y = self.data_X, self.data_Y
cv_X, cv_Y = self.cross_validation_X, self.cross_validation_Y
te_X, te_Y = self.test_X, self.test_Y
if (self.model_name == 'ann'):
prediction_probs_train, preds_train, _, acc = self.NN.get_predictions(
tr_X, True, convert_one_hot(tr_Y, self.output_classes))
prediction_probs_train = np.array(prediction_probs_train)
prediction_probs_cv, preds_cv, _, acc2 = self.NN.get_predictions(
cv_X, True, convert_one_hot(cv_Y, self.output_classes))
prediction_probs_cv = np.array(prediction_probs_cv)
prediction_probs_test, preds_test, _, acc3 = self.NN.get_predictions(
te_X, True, convert_one_hot(te_Y, self.output_classes))
prediction_probs_test = np.array(prediction_probs_test)
print("Final Accuracy of Model B on current K-1 folds of the " +
self.dataset_name + " Train " + mode + " dataset is :" +
str(acc))
print("Final Accuracy of Model B on the current fold of the " +
self.dataset_name + " Cross Validation " + mode +
" dataset is :" + str(acc2))
print("Final Accuracy of Model B on the current fold of the " +
self.dataset_name + " Test " + mode + " dataset is :" +
str(acc3))
return prediction_probs_train, prediction_probs_cv, prediction_probs_test
elif (self.model_name == 'svm' or self.model_name == 'naive_bayes'):
prediction_probs_train = np.array(
self.classifier.predict_proba(tr_X))
prediction_probs_cv = np.array(self.classifier.predict_proba(cv_X))
prediction_probs_test = np.array(
self.classifier.predict_proba(te_X))
print("Final Accuracy of Model B on current K-1 folds of the " +
self.dataset_name + " train " + mode + " dataset is :" + str(
accuracy_score(
tr_Y, np.argmax(prediction_probs_train, axis=-1))))
print("Final Accuracy of Model B on the current fold of the " +
self.dataset_name + " Cross Validation " + mode +
" dataset is :" + str(
accuracy_score(cv_Y,
np.argmax(prediction_probs_cv, axis=-1))))
print("Final Accuracy of Model B on the current fold of the " +
self.dataset_name + " Test " + mode + " dataset is :" + str(
accuracy_score(
te_Y, np.argmax(prediction_probs_test, axis=-1))))
print("Unique classes in predictions of model B on Test : ",
np.unique(np.argmax(prediction_probs_test, axis=-1)))
return prediction_probs_train, prediction_probs_cv, prediction_probs_test
elif (self.model_name == 'dt'):
prediction_probs_train = self.classifier.predict_model(tr_X)
prediction_probs_cv = self.classifier.predict_model(cv_X)
prediction_probs_test = self.classifier.predict_model(te_X)
print("Final Accuracy of Model B on current fold of" +
self.dataset_name + " CV " + mode + " Dataset is :" + str(
accuracy_score(cv_Y,
np.argmax(prediction_probs_cv, axis=-1))))
return prediction_probs_train, prediction_probs_cv, prediction_probs_test
elif (self.model_name == 'cnn'):
prediction_probs_train, preds_train, _, acc = self.CNN_classifier.get_predictions(
tr_X, True, tr_Y)
prediction_probs_cv, preds_cv, _, acc2 = self.CNN_classifier.get_predictions(
cv_X, True, cv_Y)
print("Final Accuracy of Model B on current K-1 folds of the " +
self.dataset_name + " Train " + mode + " dataset is :" +
str(acc))
print("Final Accuracy of Model B on the current fold of the " +
self.dataset_name + " Cross Validation " + mode +
" dataset is :" + str(acc2))
prediction_probs_test, preds_test, _ = self.CNN_classifier.get_predictions(
te_X, False, te_Y)
return prediction_probs_train, prediction_probs_cv, prediction_probs_test
elif (self.model_name == 'inceptionv3'):
prediction_probs_train, preds_train, _, acc = self.inception_classifier.get_output(
tr_X, True, tr_Y)
prediction_probs_cv, preds_cv, _, acc2 = self.inception_classifier.get_output(
cv_X, True, cv_Y)
print("Final Accuracy of Model B on current K-1 folds of the " +
self.dataset_name + " Train " + mode + " dataset is :" +
str(acc))
print("Final Accuracy of Model B on the current fold of the " +
self.dataset_name + " Cross Validation " + mode +
" dataset is :" + str(acc2))
prediction_probs_test, preds_test, _ = self.inception_classifier.get_output(
te_X, False, te_Y)
return prediction_probs_train, prediction_probs_cv, prediction_probs_test
else:
print("Not yet implemented")
return None, None, None
class ModelA(object):
def __init__(self, epochs, batch_size, dataset_name, learning_rate,
model_name, is_binarized, is_resized, is_grayscale):
self.dataset_name = dataset_name
self.learning_rate = learning_rate
self.model_name = model_name
self.no_epochs = epochs
self.batch_size = batch_size
self.is_resized = is_resized
self.is_grayscale = is_grayscale
self.is_binarized = is_binarized
if (dataset_name == 'mnist'):
if (self.is_resized):
self.input_image_size = (10, 10, 1)
else:
self.input_image_size = (28, 28, 1)
if (is_binarized):
self.output_classes = 2
else:
self.output_classes = 10
elif (dataset_name == 'fashion_mnist'):
self.input_image_size = (10, 10,
1) if (self.is_resized) else (28, 28, 1)
self.output_classes = 2 if (self.is_binarized) else 10
elif (dataset_name == 'stanford40'):
self.input_image_size = (200, 200, 3)
self.output_classes = 40
else:
print("Not Implemented yet")
if (self.model_name == "svm"):
self.classifier = svm.SVC(C=1,
kernel='rbf',
gamma='auto',
probability=True,
random_state=0)
self.classifier = fit_model_to_initial_dataset(
self.dataset_name, self.classifier, self.model_name,
self.is_resized, self.is_grayscale)
elif (self.model_name == "naive_bayes"):
self.classifier = MultinomialNB(alpha=1.0)
self.classifier = fit_model_to_initial_dataset(
self.dataset_name, self.classifier, self.model_name,
self.is_resized, self.is_grayscale)
elif (self.model_name == "ann"):
input_size = np.prod(self.input_image_size)
self.hidden_layers = hidden_layer_list_model_A
self.NN = Neural_Network(self.no_epochs, self.batch_size,
self.learning_rate, input_size,
self.output_classes, self.hidden_layers)
self.NN.create_tf_model("ModelA")
elif (self.model_name == "dt"):
self.classifier = DecisionTree('gini', 'best', None, 5)
self.classifier = fit_model_to_initial_dataset(
self.dataset_name, self.classifier, "dt", self.is_resized,
self.is_grayscale)
elif (self.model_name == "ensemble"):
input_size = np.prod(self.input_image_size)
self.classifier = ensemble_model(input_size, self.no_epochs,
self.batch_size,
self.output_classes,
self.learning_rate, "")
self.classifier.initialize_ensemble(ensemble_list,
self.dataset_name,
self.is_resized,
self.is_grayscale)
print(ensemble_list)
elif (self.model_name == "cnn"):
self.CNN_classifier = CNN(self.input_image_size, self.no_epochs,
self.batch_size, self.output_classes,
self.learning_rate)
self.CNN_classifier.initialize_model()
elif (self.model_name == "inceptionv3"):
self.inception_classifier = Inceptionv3(self.output_classes,
self.batch_size,
self.no_epochs,
self.input_image_size)
self.inception_classifier.initialize_model()
def set_dataset(self, train_X, train_Y, test_X, test_Y, cv_X, cv_Y):
self.data_X, self.data_Y, self.test_X, self.test_Y, self.cross_validation_X, self.cross_validation_Y = train_X, train_Y, test_X, test_Y, cv_X, cv_Y
def calculate_interpretability(self,
probs_model_B_train,
probs_model_B_cross_validation,
probs_model_B_test,
probs_B_train_cf=None,
probs_B_cv_cf=None,
probs_B_test_cf=None):
if (self.model_name == 'ann' or self.model_name == 'svm'
or self.model_name == 'dt' or self.model_name == "ensemble"
or self.model_name == "naive_bayes"):
self.data_X = self.data_X.reshape(self.data_X.shape[0], -1)
self.cross_validation_X = self.cross_validation_X.reshape(
self.cross_validation_X.shape[0], -1)
self.test_X = self.test_X.reshape(self.test_X.shape[0], -1)
initial_entropy_train = self.calculate_entropy(probs_model_B_train,
'initial', 0)
initial_entropy_cv = self.calculate_entropy(
probs_model_B_cross_validation, 'initial', 1)
initial_entropy_test = self.calculate_entropy(probs_model_B_test,
'initial', 2)
self.initialize_model(probs_model_B_train,
probs_model_B_cross_validation)
final_entropy_train = self.calculate_entropy(probs_model_B_train,
'final', 0)
final_entropy_cv = self.calculate_entropy(
probs_model_B_cross_validation, 'final', 1)
final_entropy_test = self.calculate_entropy(probs_model_B_test,
'final', 2)
print("Initial entropies: ", initial_entropy_train, initial_entropy_cv,
initial_entropy_test)
print("Final entropies: ", final_entropy_train, final_entropy_cv,
final_entropy_test)
print("The initial entropy on Train Dataset is :",
initial_entropy_train)
print("The initial entropy on Cross Validation Dataset is :",
initial_entropy_cv)
print("The final entropy on Train Dataset is : ", final_entropy_train)
print("The final entropy on Cross Validation Dataset is : ",
final_entropy_cv)
if (initial_entropy_train == 0.0):
interpret_train = 1.0
else:
interpret_train = (initial_entropy_train -
final_entropy_train) / initial_entropy_train
if (initial_entropy_cv == 0.0):
interpret_cv = 1.0
else:
interpret_cv = (initial_entropy_cv -
final_entropy_cv) / initial_entropy_cv
if (initial_entropy_test == 0.0):
interpret_test = 1.0
else:
interpret_test = (initial_entropy_test -
final_entropy_test) / initial_entropy_test
#re-initialize the classifier to original state for SVM and Decision Tree.
if (self.model_name == 'svm' or self.model_name == 'dt'
or self.model_name == 'naive_bayes'):
self.classifier = fit_model_to_initial_dataset(
self.dataset_name, self.classifier, self.model_name,
self.is_resized, self.is_grayscale)
elif (self.model_name == "ensemble"):
self.classifier.fit_model(self.is_resized, self.is_grayscale)
return interpret_train, interpret_cv, interpret_test
def initialize_model(self, probs_model_B_train, probs_model_B_cv):
predictions_model_B_train = np.argmax(probs_model_B_train, axis=-1)
predictions_model_B_cv = np.argmax(probs_model_B_cv, axis=-1)
data_train = self.data_X
data_cv = self.cross_validation_X
if (self.model_name == 'svm' or self.model_name == 'naive_bayes'):
print(data_train.shape, predictions_model_B_train.shape)
self.classifier.fit(data_train, predictions_model_B_train)
print("Fitted the" + self.model_name +
" to the Predictions of Model B")
elif (self.model_name == 'ann'):
data_train = data_train.reshape(data_train.shape[0], -1)
print(data_train.shape, predictions_model_B_train.shape,
data_cv.shape)
self.NN.train_model(data_train, predictions_model_B_train, data_cv,
predictions_model_B_cv)
print(
"Trained the Neural Network Model A on Predictions of Model B")
elif (self.model_name == 'dt'):
data_train = data_train.reshape(data_train.shape[0], -1)
data_cv = data_cv.reshape(data_cv.shape[0], -1)
self.classifier.train_model(data_train, predictions_model_B_train,
data_cv, predictions_model_B_cv)
elif (self.model_name == "ensemble"):
data_train = data_train.reshape(data_train.shape[0], -1)
data_cv = data_cv.reshape(data_cv.shape[0], -1)
self.classifier.train_ensemble(data_train,
predictions_model_B_train, data_cv,
predictions_model_B_cv)
elif (self.model_name == "cnn"):
self.CNN_classifier.train_model(data_train,
predictions_model_B_train, data_cv,
predictions_model_B_cv)
elif (self.model_name == "inceptionv3"):
self.inception_classifier.train_model(data_train,
predictions_model_B_train,
data_cv,
predictions_model_B_cv)
else:
print("Not implemented yet")
def calculate_entropy(self, probs_B, name, split):
preds = np.argmax(probs_B, axis=-1)
if (split == 0):
data = self.data_X
output = self.data_Y
#print("Train split")
elif (split == 1):
data = self.cross_validation_X
output = self.cross_validation_Y
#print("Cross Validation split ")
elif (split == 2):
data = self.test_X
output = self.test_Y
#print("Test split ")
else:
#print("Invalid Split Value")
return None
if (self.model_name == 'svm' or self.model_name == 'naive_bayes'):
probs2 = np.array(self.classifier.predict_proba(data))
probs = convert_to_all_classes_array(probs2,
self.classifier.classes_,
self.output_classes)
if (probs2.shape[-1] == self.output_classes):
assert (probs2.all() == probs.all())
print(
"Accuracy of Model A on the current split of the dataset is : ",
accuracy_score(output, np.argmax(probs, axis=-1)))
elif (self.model_name == 'ann'):
probs, _, _, acc = self.NN.get_predictions(
data, True, convert_one_hot(
output, self.output_classes)) # These are 1X50000 arrays
print(probs.shape)
print("Accuracy of Model A on the" + self.dataset_name +
"Training Dataset is: " + str(acc))
probs = np.array(probs)
if (probs.shape[0] == 1):
probs = np.squeeze(probs, axis=0)
elif (self.model_name == 'cnn' or self.model_name == 'inceptionv3'):
if (self.model_name == 'inceptionv3'):
probs, _, _, acc = self.inception_classifier.get_output(
data, True, output)
else:
probs, _, _, acc = self.CNN_classifier.get_predictions(
data, True, output)
print("Accuracy of Model A on the" + self.dataset_name +
"Training Dataset is: " + str(acc))
probs = np.array(probs)
if (probs.shape[0] == 1):
probs = np.squeeze(probs, axis=0)
elif (self.model_name == 'dt'):
probs2 = self.classifier.predict_model(data)
probs = convert_to_all_classes_array(probs2,
self.classifier.classes_,
self.output_classes)
if (probs2.shape[-1] == self.output_classes):
assert (probs2.all() == probs.all())
elif (self.model_name == "ensemble"):
probs = np.array(
self.classifier.predict_ensemble(
data, True, convert_one_hot(output, self.output_classes)))
prob_A_indexes = np.argmax(probs, axis=-1)
print(
"Accuracy of Model A on the current split of the predictions of Model B of the dataset is: ",
accuracy_score(preds, prob_A_indexes))
total_diff = 0.0
list_diff = []
for i in range(probs.shape[0]):
if (prob_A_indexes[i] != preds[i]):
list_diff.append([i, prob_A_indexes[i], preds[i]])
val = (abs(probs[i][prob_A_indexes[i]] - probs[i][preds[i]]))
if (val <= 0):
total_diff += 0.0
else:
total_diff += -1.0 * (math.log2(val))
total_diff = (total_diff) / (probs.shape[0])
if (len(list_diff) == 0):
print(
"For Model A " + str(self.model_name) +
" and Model B, the final predictions on this split of the dataset are same"
)
else:
None
return total_diff
def dump_data(self, data_file, **kwargs):
arguments = ''
for key, val in kwargs.items():
arguments += str(val['name'])
arguments += "=kwargs['" + str(key) + "']['val'], "
arguments = arguments.strip()
arguments = arguments[:-1]
eval("np.savez_compressed(data_file, " + arguments + ")")
print("Data dumped")
class ModelB(object):
def __init__(self, sess, epochs, batch_size, dataset_name, learning_rate, model_name, is_binarized, is_resized, is_grayscale, feature_size = None):
self.sess = sess
self.dataset_name = dataset_name
self.learning_rate = learning_rate
self.model_name = model_name
self.no_epochs = epochs
self.batch_size = batch_size
self.is_binarized = is_binarized
self.is_resized = is_resized
self.is_grayscale = is_grayscale
self.no_features = feature_size
if (dataset_name == 'mnist'):
if(self.is_resized):
self.input_image_size = (10, 10, 1)
else:
self.input_image_size = (28, 28, 1)
if (is_binarized):
self.output_classes = 2
else:
self.output_classes = 10
# self.data_X, self.data_Y, self.test_X, self.test_Y = load_mnist(is_binarized)
# self.cross_validation_X, self.cross_validation_Y = self.data_X, self.data_Y
# self.no_batches = len(self.data_X)//(self.batch_size)
elif (dataset_name == 'cifar-10'):
if(self.is_grayscale):
self.input_image_size = (28, 28, 1) if (self.is_resized) else (32, 32, 1)
else:
self.input_image_size = (32, 32, 3)
self.output_classes = 10
elif (dataset_name == 'fashion_mnist'):
self.input_image_size = (10, 10, 1) if (self.is_resized) else (28, 28, 1)
self.output_classes = 2 if (self.is_binarized) else 10
elif(dataset_name == 'stanford40'):
self.input_image_size = (200, 200, 3)
self.output_classes = 40
elif(dataset_name == 'sentiment_analysis'):
self.input_image_size = (self.no_features, )
self.output_classes = 2
else:
print("Not Implemented yet")
if (self.model_name == 'ann'):
input_size = np.prod(self.input_image_size)
self.hidden_layers = [512, 256, 128, 64]
self.NN = Neural_Network(self.sess, self.no_epochs, self.batch_size, self.learning_rate, input_size,
self.output_classes, self.hidden_layers, mc_dropout = False, dropout_rate = None)
elif (self.model_name == 'svm'):
self.classifier = svm.SVC(C=1, kernel='rbf', gamma='auto', probability=True, random_state=0)
#self.classifier = SVC(kernel = 'rbf', gamma = 'auto', C = 1, probability = True, verbose = True, random_state = 0)
elif(self.model_name == 'dt'):
self.classifier = DecisionTree('gini', 'best', None, 10)
elif(self.model_name == 'cnn'):
self.CNN_classifier = CNN(self.input_image_size, self.no_epochs, self.batch_size, self.output_classes, self.learning_rate)
elif(self.model_name == 'inceptionv3'):
self.inception_classifier = Inceptionv3(self.output_classes, self.batch_size, self.no_epochs, self.input_image_size)
#print("Parameters Initialized")
def set_dataset(self, data_X, data_Y, test_X, test_Y, cross_validation_X, cross_validation_Y):
self.data_X, self.data_Y, self.test_X, self.test_Y, self.cross_validation_X, self.cross_validation_Y = data_X, data_Y, test_X, test_Y, cross_validation_X, cross_validation_Y
self.no_batches = len(self.data_X) // (self.batch_size)
def init_model(self):
if (self.model_name == 'ann'):
self.NN.create_tf_model("ModelB")
elif (self.model_name == 'cnn'):
self.CNN_classifier.initialize_model()
elif(self.model_name == 'inceptionv3'):
self.inception_classifier.initialize_model()
elif (self.model_name == 'svm'):
pass
elif(self.model_name == 'dt'):
pass
else:
print("Not implemented yet")
##print("Model Initialized")
def train_model(self):
if(self.model_name == 'cnn'):
self.CNN_classifier.train_model(self.data_X, self.data_Y, self.cross_validation_X, self.cross_validation_Y)
elif(self.model_name == 'inceptionv3'):
self.inception_classifier.train_model(self.data_X, self.data_Y, self.cross_validation_X, self.cross_validation_Y)
else:
self.data_X = self.data_X.reshape(self.data_X.shape[0], -1)
self.cross_validation_X = self.cross_validation_X.reshape(self.cross_validation_X.shape[0], -1)
self.test_X = self.test_X.reshape(self.test_X.shape[0], -1)
if (self.model_name == 'svm'):
#TODO::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::ADD CROSS VALIDATION HERE
print(self.data_Y.shape)
self.classifier.fit(self.data_X, self.data_Y)
print("Training Finished")
elif (self.model_name == 'ann'):
self.NN.train_model(self.data_X, self.data_Y, self.cross_validation_X, self.cross_validation_Y)
elif(self.model_name == 'dt'):
self.classifier.train_model(self.data_X, self.data_Y, self.cross_validation_X, self.cross_validation_Y)
else:
print("Not yet implemented")
def get_output(self):
if (self.model_name == 'ann'):
prediction_probs_train, preds_train, _, acc = self.NN.get_predictions(self.data_X, True,
convert_one_hot(self.data_Y,
self.output_classes))
prediction_probs_train = np.array(prediction_probs_train)
print(prediction_probs_train.shape)
# prediction_probs = prediction_probs.reshape((prediction_probs.shape[1], prediction_probs.shape[2]))
prediction_probs_train = np.argmax(prediction_probs_train, axis=-1)
prediction_probs_cv, preds_cv, _, acc2 = self.NN.get_predictions(self.cross_validation_X, True,
convert_one_hot(self.cross_validation_Y, self.output_classes))
prediction_probs_cv = np.argmax(np.array(prediction_probs_cv), axis=-1)
prediction_probs_test, preds_test, _ , _ = self.NN.get_predictions(self.test_X, True, convert_one_hot(self.test_Y, self.output_classes))
prediction_probs_test = np.argmax(np.array(prediction_probs_test), axis=-1)
print("Final Accuracy of Model B on current K-1 folds of the " + self.dataset_name + " Train dataset is :" + str(acc))
print("Final Accuracy of Model B on the current fold of the " + self.dataset_name + " Cross Validation dataset is :" + str(acc2))
#a = input()
# print(prediction_probs, self.new_data_Y)
print(np.unique(prediction_probs_train), np.unique(prediction_probs_cv), np.unique(prediction_probs_test))
return prediction_probs_train, prediction_probs_cv, prediction_probs_test
elif (self.model_name == 'svm'):
prediction_probs_train = np.array(self.classifier.predict_proba(self.data_X))
prediction_probs_cv = np.array(self.classifier.predict_proba(self.cross_validation_X))
prediction_probs_test = np.array(self.classifier.predict_proba(self.test_X))
prediction_probs_train = np.argmax(prediction_probs_train, axis=-1)
prediction_probs_cv = np.argmax(prediction_probs_cv, axis=-1)
prediction_probs_test = np.argmax(prediction_probs_test, axis = -1)
# preds_train = self.classifier.predict(self.data_X)
# preds_cv = self.classifier.predict(self.cross_validation_X)
print("Final Accuracy of Model B on current K-1 folds of the " + self.dataset_name + " train dataset is :" + str(accuracy_score(self.data_Y, prediction_probs_train)))
print("Final Accuracy of Model B on the current fold of the " + self.dataset_name + " Cross Validation dataset is :" + str(accuracy_score(self.cross_validation_Y, prediction_probs_cv)))
a = input()
return prediction_probs_train, prediction_probs_cv, prediction_probs_test
elif(self.model_name == 'dt'):
prediction_probs_train = np.argmax(self.classifier.predict_model(self.data_X), axis = -1)
prediction_probs_cv = np.argmax(self.classifier.predict_model(self.cross_validation_X), axis = -1)
prediction_probs_test = np.argmax(self.classifier.predict_model(self.test_X), axis = -1)
print("Final Accuracy of Model B on current fold of" + self.dataset_name + " CV Dataset is :" + str(accuracy_score(self.cross_validation_Y, prediction_probs_cv)))
return prediction_probs_train, prediction_probs_cv, prediction_probs_test
elif(self.model_name == 'cnn'):
prediction_probs_train, preds_train, _, acc = self.CNN_classifier.get_predictions(self.data_X, True, self.data_Y)
#prediction_probs_train = np.argmax(np.array(prediction_probs_train), axis = -1)
prediction_probs_cv, preds_cv, _, acc2 = self.CNN_classifier.get_predictions(self.cross_validation_X, True, self.cross_validation_Y)
print("Final Accuracy of Model B on current K-1 folds of the " + self.dataset_name + " Train dataset is :" + str(acc))
print("Final Accuracy of Model B on the current fold of the " + self.dataset_name + " Cross Validation dataset is :" + str(acc2))
prediction_probs_test, preds_test, _ = self.CNN_classifier.get_predictions(self.test_X, False, self.test_Y)
return preds_train, preds_cv, preds_test
elif(self.model_name == 'inceptionv3'):
prediction_probs_train, preds_train, _, acc = self.inception_classifier.get_output(self.data_X, True, self.data_Y)
prediction_probs_cv, preds_cv, _, acc2 = self.inception_classifier.get_output(self.cross_validation_X, True, self.cross_validation_Y)
print("Final Accuracy of Model B on current K-1 folds of the " + self.dataset_name + " Train dataset is :" + str(acc))
print("Final Accuracy of Model B on the current fold of the " + self.dataset_name + " Cross Validation dataset is :" + str(acc2))
prediction_probs_test, preds_test, _ = self.inception_classifier.get_output(self.test_X, False, self.test_Y)
return preds_train, preds_cv, preds_test
else:
print("Not yet implemented")
return None, None, None
class ModelA(object):
def __init__(self,
epochs,
batch_size,
dataset_name,
learning_rate,
model_name,
is_binarized,
is_resized,
is_grayscale,
feature_size=None,
mode='original',
entropy_mode='original'):
self.dataset_name = dataset_name
self.learning_rate = learning_rate
self.model_name = model_name
self.no_epochs = epochs
self.batch_size = batch_size
self.is_resized = is_resized
self.is_grayscale = is_grayscale
# self.classifier = SGDClassifier(loss = "hinge", penalty = "l2", max_iter = 5)
self.is_binarized = is_binarized
self.no_features = feature_size
self.interpretability_mode = mode
self.entropy_mode = entropy_mode
if (dataset_name == 'mnist'):
if (self.is_resized):
self.input_image_size = (10, 10, 1)
else:
self.input_image_size = (28, 28, 1)
if (is_binarized):
self.output_classes = 2
else:
self.output_classes = 10
# self.data_X, self.data_Y, self.test_X, self.test_Y = load_mnist_dataset(self.is_binarized)
elif (dataset_name == 'cifar-10'):
if (self.is_grayscale):
self.input_image_size = (28, 28,
1) if (self.is_resized) else (32, 32,
1)
else:
self.input_image_size = (32, 32, 3)
self.output_classes = 10
elif (dataset_name == 'fashion_mnist'):
self.input_image_size = (10, 10,
1) if (self.is_resized) else (28, 28, 1)
self.output_classes = 2 if (self.is_binarized) else 10
elif (dataset_name == 'stanford40'):
self.input_image_size = (200, 200, 3)
self.output_classes = 40
elif (dataset_name == 'sentiment_analysis'):
self.input_image_size = (self.no_features, )
self.output_classes = 2 if (self.is_binarized) else 5
else:
print("Not Implemented yet")
if (self.model_name == "svm"):
if (self.dataset_name == 'sentiment_analysis'):
#param_list = svm_parameter_list.copy()
#self.param_list = {'C': 1, 'kernel' : 'linear'}
#self.param_list = {'C': 0.1, 'kernel' : 'poly', 'gamma' : 10, 'coef0': 2.0, 'degree' : 2}
self.param_list = {'C': 10, 'kernel': 'rbf', 'gamma': 1}
self.param_list['probability'] = True
self.param_list['random_state'] = 0
print(self.param_list)
self.classifier = svm.SVC(
C=self.param_list['C'],
kernel=self.param_list['kernel'],
gamma=self.param_list['gamma'],
probability=self.param_list['probability'],
random_state=self.param_list['random_state'],
decision_function_shape='ovo')
else:
self.classifier = svm.SVC(C=1,
kernel='rbf',
gamma='auto',
probability=True,
random_state=0)
#self.classifier = SVC(kernel = 'rbf', gamma = 'auto', C = 1, probability = True, verbose = True, random_state = 0)
self.classifier = fit_model_to_initial_dataset(
self.dataset_name, self.classifier, self.model_name,
self.is_resized, self.is_grayscale)
elif (self.model_name == "naive_bayes"):
if (self.dataset_name == 'sentiment_analysis'):
self.classifier = MultinomialNB(
alpha=nb_parameter_list['alpha'])
else:
self.classifier = MultinomialNB(alpha=1.0)
self.classifier = fit_model_to_initial_dataset(
self.dataset_name, self.classifier, self.model_name,
self.is_resized, self.is_grayscale)
elif (self.model_name == "ann"):
input_size = np.prod(self.input_image_size)
self.hidden_layers = [256]
self.NN = Neural_Network(self.no_epochs, self.batch_size,
self.learning_rate, input_size,
self.output_classes, self.hidden_layers)
self.NN.create_tf_model("ModelA")
#self.fit_model_to_initial_dataset()
elif (self.model_name == "dt"):
self.classifier = DecisionTree('gini', 'best', None, 5)
print("Gini_Best_5")
self.classifier = fit_model_to_initial_dataset(
self.dataset_name, self.classifier, "dt", self.is_resized,
self.is_grayscale)
elif (self.model_name == "ensemble"):
input_size = np.prod(self.input_image_size)
self.classifier = ensemble_model(input_size, self.no_epochs,
self.batch_size,
self.output_classes,
self.learning_rate, "")
if (self.dataset_name == 'sentiment_analysis'):
param_list = svm_parameter_list.copy()
param_list['probability'] = True
param_list['random_state'] = 0
param_list['gamma'] = 'scale'
ensemble_list = [("svm", param_list), ("lr", ),
("dt", "gini", "best", None, 10)]
else:
param_list = {
'C': 0.01,
'kernel': 'rbf',
'gamma': 'auto',
'probability': True,
'random_state': 0
}
ensemble_list = [("svm", param_list), ("lr", ),
("dt", "gini", "best", None, 10)]
self.classifier.initialize_ensemble(ensemble_list,
self.dataset_name,
self.is_resized,
self.is_grayscale)
print(ensemble_list)
#("svm", ), ("ann", [512, 128]), ("knn", ),
#
#self.classifier.initialize_ensemble([("svm", ), ("dt", "gini", "best", None, 10)], self.dataset_name, self.is_resized, self.is_grayscale)
#print("Ensemble being used: ann of 64 neurons\n")
elif (self.model_name == "cnn"):
self.CNN_classifier = CNN(self.input_image_size, self.no_epochs,
self.batch_size, self.output_classes,
self.learning_rate)
self.CNN_classifier.initialize_model()
elif (self.model_name == "inceptionv3"):
self.inception_classifier = Inceptionv3(self.output_classes,
self.batch_size,
self.no_epochs,
self.input_image_size)
self.inception_classifier.initialize_model()
def set_dataset(self, train_X, train_Y, test_X, test_Y, cv_X, cv_Y):
self.data_X, self.data_Y, self.test_X, self.test_Y, self.cross_validation_X, self.cross_validation_Y = train_X, train_Y, test_X, test_Y, cv_X, cv_Y
def set_counter_factual_dataset(self, data_X, data_Y, test_X, test_Y,
cross_validation_X, cross_validation_Y):
self.cf_data_X, self.cf_data_Y, self.cf_test_X, self.cf_test_Y, self.cf_cv_X, self.cf_cv_Y = data_X, data_Y, test_X, test_Y, cross_validation_X, cross_validation_Y
def calculate_interpretability(self,
probs_model_B_train,
probs_model_B_cross_validation,
probs_model_B_test,
probs_B_train_cf=None,
probs_B_cv_cf=None,
probs_B_test_cf=None):
if (self.model_name == 'ann' or self.model_name == 'svm'
or self.model_name == 'dt' or self.model_name == "ensemble"
or self.model_name == "naive_bayes"):
self.data_X = self.data_X.reshape(self.data_X.shape[0], -1)
self.cross_validation_X = self.cross_validation_X.reshape(
self.cross_validation_X.shape[0], -1)
self.test_X = self.test_X.reshape(self.test_X.shape[0], -1)
if (self.interpretability_mode == 'counter_factual'):
self.cf_data_X = self.cf_data_X.reshape(
self.cf_data_X.shape[0], -1)
self.cf_cv_X = self.cf_cv_X.reshape(self.cf_cv_X.shape[0], -1)
self.cf_test_X = self.cf_test_X.reshape(
self.cf_test_X.shape[0], -1)
##print("The size of the final dataset used for interpretation is: ", self.data_X.shape)
if (self.interpretability_mode == 'counter_factual'):
print("Entropy calculated on counter factual data")
initial_entropy_train = self.calculate_entropy(
probs_B_train_cf, 'initial', 0)
initial_entropy_cv = self.calculate_entropy(
probs_B_cv_cf, 'initial', 1)
initial_entropy_test = self.calculate_entropy(
probs_B_test_cf, 'initial', 2)
elif (self.interpretability_mode == 'original'):
initial_entropy_train = self.calculate_entropy(
probs_model_B_train, 'initial', 0)
initial_entropy_cv = self.calculate_entropy(
probs_model_B_cross_validation, 'initial', 1)
initial_entropy_test = self.calculate_entropy(
probs_model_B_test, 'initial', 2)
#initial_entropy_train, initial_entropy_test = None, None
#print(initial_entropy_train, initial_entropy_cv, initial_entropy_test)
#compute WAIC for model A for SVMs
if (self.model_name == 'naive_bayes'):
if (self.interpretability_mode == 'counter_factual'):
if (self.entropy_mode == 'counter_factual'):
waic_tr_data_X = self.cf_data_X
waic_tr_data_Y = probs_B_train_cf
elif (self.entropy_mode == 'original'):
waic_tr_data_X = self.data_X
waic_tr_data_Y = probs_model_B_train
waic_calc_data_X = self.cf_data_X
waic_calc_data_Y = probs_B_train_cf
waic_test_X = self.cf_test_X
waic_test_Y = probs_B_test_cf
elif (self.interpretability_mode == 'original'):
waic_tr_data_X = self.data_X
waic_test_X = self.test_X
waic_calc_data_X = self.data_X
waic_calc_data_Y = probs_model_B_train
waic_tr_data_Y = probs_model_B_train
waic_test_Y = probs_model_B_test
params = self.param_list.copy()
waic_tr_data_Y = np.argmax(waic_tr_data_Y, axis=-1)
waic_calc_data_Y = np.argmax(waic_calc_data_Y, axis=-1)
waic_test_Y = np.argmax(waic_test_Y, axis=-1)
print("Unique Classes in train_data for WAIC",
np.unique(waic_tr_data_Y))
print("Unique Classes in calc_data for WAIC",
np.unique(waic_calc_data_Y))
print("Unique Classes in test_data for WAIC",
np.unique(waic_test_Y))
del params['kernel']
waic_train_per_class, waic_test_per_class, waic_train_total, waic_test_total = final_computation_WAIC(
self.param_list['kernel'], waic_tr_data_X, waic_tr_data_Y,
waic_calc_data_X, waic_calc_data_Y, waic_test_X, waic_test_Y,
self.classifier, params)
print("Per class WAIC values: Train: ", waic_train_per_class,
" Test: ", waic_test_per_class)
print("Overall WAIC values: Train: ", waic_train_total, " Test: ",
waic_test_total)
if (self.interpretability_mode == 'counter_factual'):
if (self.entropy_mode == 'counter_factual'):
self.initialize_model(probs_B_train_cf, probs_B_test_cf)
elif (self.entropy_mode == 'original'):
self.initialize_model(probs_model_B_train,
probs_model_B_cross_validation)
elif (self.interpretability_mode == 'original'):
self.initialize_model(probs_model_B_train,
probs_model_B_cross_validation)
if (self.interpretability_mode == 'original'):
final_entropy_train = self.calculate_entropy(
probs_model_B_train, 'final', 0)
final_entropy_cv = self.calculate_entropy(
probs_model_B_cross_validation, 'final', 1)
final_entropy_test = self.calculate_entropy(
probs_model_B_test, 'final', 2)
elif (self.interpretability_mode == 'counter_factual'):
print("Entropy calculated on counter factual data")
final_entropy_train = self.calculate_entropy(
probs_B_train_cf, 'final', 0)
final_entropy_cv = self.calculate_entropy(probs_B_cv_cf, 'final',
1)
final_entropy_test = self.calculate_entropy(
probs_B_test_cf, 'final', 2)
#final_entropy_train, final_entropy_test = None, None
#a = input()
print(final_entropy_train, final_entropy_cv, final_entropy_test)
print("The initial entropy on Train Dataset is :",
initial_entropy_train)
print("The initial entropy on Cross Validation Dataset is :",
initial_entropy_cv)
print("The final entropy on Train Dataset is : ", final_entropy_train)
print("The final entropy on Cross Validation Dataset is : ",
final_entropy_cv)
if (initial_entropy_train == 0.0):
interpret_train = 1.0
else:
interpret_train = (initial_entropy_train -
final_entropy_train) / initial_entropy_train
if (initial_entropy_cv == 0.0):
interpret_cv = 1.0
else:
interpret_cv = (initial_entropy_cv -
final_entropy_cv) / initial_entropy_cv
if (initial_entropy_test == 0.0):
interpret_test = 1.0
else:
interpret_test = (initial_entropy_test -
final_entropy_test) / initial_entropy_test
#self.dump_file_handle.close()
#re-initialize the classifier to original state for SVM and Decision Tree.
#This is necessary to ensure proper cross-validation.
if (self.model_name == 'svm' or self.model_name == 'dt'
or self.model_name == 'naive_bayes'):
self.classifier = fit_model_to_initial_dataset(
self.dataset_name, self.classifier, self.model_name,
self.is_resized, self.is_grayscale)
elif (self.model_name == "ensemble"):
self.classifier.fit_model(self.is_resized, self.is_grayscale)
return interpret_train, interpret_cv, interpret_test
def initialize_model(self, probs_model_B_train, probs_model_B_cv):
predictions_model_B_train = np.argmax(probs_model_B_train, axis=-1)
predictions_model_B_cv = np.argmax(probs_model_B_cv, axis=-1)
if (self.interpretability_mode == 'original'):
data_train = self.data_X
data_cv = self.cross_validation_X
elif (self.interpretability_mode == 'counter_factual'):
if (self.entropy_mode == 'counter_factual'):
data_train = self.cf_data_X
data_cv = self.cf_cv_X
elif (self.entropy_mode == 'original'):
data_train = self.data_X
data_cv = self.cross_validation_X
if (self.model_name == 'svm' or self.model_name == 'naive_bayes'):
# self.classifier = SGDClassifier(loss = "hinge")
print(data_train.shape, predictions_model_B_train.shape)
#self.classifier.fit(self.data_X[:10000], predictions_model_B[:10000]) CHANGE
#TODO::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::ADD CROSS VALIDATION HERE
self.classifier.fit(data_train, predictions_model_B_train)
print("Fitted the" + self.model_name +
" to the Predictions of Model B")
elif (self.model_name == 'ann'):
data_train = data_train.reshape(data_train.shape[0], -1)
print(data_train.shape, predictions_model_B_train.shape,
data_cv.shape)
self.NN.train_model(data_train, predictions_model_B_train, data_cv,
predictions_model_B_cv)
print(
"Trained the Neural Network Model A on Predictions of Model B")
elif (self.model_name == 'dt'):
data_train = data_train.reshape(data_train.shape[0], -1)
data_cv = data_cv.reshape(data_cv.shape[0], -1)
#preds2 = np.argmax(self.classifier.predict_model(data_train), axis = -1)
#print("Final Accuracy of Model A on MNIST Train Dataset before interpretation is :" + str(metrics.accuracy_score(self.data_Y, preds2)))
self.classifier.train_model(data_train, predictions_model_B_train,
data_cv, predictions_model_B_cv)
#print("Fitted the Decision Tree to the predictions of model B")
#preds = np.argmax(self.classifier.predict_model(data_cv), axis = -1)
#print("Final Accuracy of Model A on MNIST CV Dataset is :" + str(metrics.accuracy_score(self.cross_validation_Y, preds)))
#preds2 = np.argmax(self.classifier.predict_model(data_train), axis = -1)
#print("Final Accuracy of Model A on MNIST CV Dataset is :" + str(metrics.accuracy_score(self.data_Y, preds2)))
elif (
self.model_name == "ensemble"
): #cannot have CNN here, as the data_X is being flattened out here, for use by other models
data_train = data_train.reshape(data_train.shape[0], -1)
data_cv = data_cv.reshape(data_cv.shape[0], -1)
self.classifier.train_ensemble(data_train,
predictions_model_B_train, data_cv,
predictions_model_B_cv)
elif (self.model_name == "cnn"):
self.CNN_classifier.train_model(data_train,
predictions_model_B_train, data_cv,
predictions_model_B_cv)
elif (self.model_name == "inceptionv3"):
self.inception_classifier.train_model(data_train,
predictions_model_B_train,
data_cv,
predictions_model_B_cv)
else:
print("Not implemented yet")
def calculate_entropy(self, probs_B, name, split):
# prob = np.array(self.classifier.decision_function(self.data_X))
# prob_B_indexes = np.argmax(predictions_model_B, axis = -1)
preds = np.argmax(probs_B, axis=-1)
if (split == 0):
if (self.interpretability_mode == 'original'):
data = self.data_X
output = self.data_Y
elif (self.interpretability_mode == 'counter_factual'):
data = self.cf_data_X
output = self.cf_data_Y
#print("Train split")
elif (split == 1):
if (self.interpretability_mode == 'original'):
data = self.cross_validation_X
output = self.cross_validation_Y
elif (self.interpretability_mode == 'counter_factual'):
data = self.cf_cv_X
output = self.cf_cv_Y
#print("Cross Validation split ")
elif (split == 2):
if (self.interpretability_mode == 'original'):
data = self.test_X
output = self.test_Y
elif (self.interpretability_mode == 'counter_factual'):
data = self.cf_test_X
output = self.cf_test_Y
#print("Test split")
else:
#print("Invalid Split Value")
return None
if (self.model_name == 'svm' or self.model_name == 'naive_bayes'):
#probs_train = np.array(self.classifier.predict_proba(self.data_X[:10000])) CHANGE
probs2 = np.array(self.classifier.predict_proba(data))
print(probs2.shape)
probs = convert_to_all_classes_array(probs2,
self.classifier.classes_,
self.output_classes)
if (probs2.shape[-1] == self.output_classes):
assert (probs2.all() == probs.all())
#print(probs)
#a = input()
######CHANGE MADE IN LINE BELOW, CONFIRM : Earlier: probs2, Now: probs
categorical_outputs = to_categorical(output)
print(
"Accuracy of Model A on the current split of the dataset is : ",
accuracy_score(np.argmax(categorical_outputs, axis=-1),
np.argmax(probs, axis=-1)))
elif (self.model_name == 'ann'):
probs, _, _, acc = self.NN.get_predictions(
data, True, convert_one_hot(
output, self.output_classes)) # These are 1X50000 arrays
print(probs.shape)
print("Accuracy of Model A on the" + self.dataset_name +
"Training Dataset is: " + str(acc))
#print("Accuracy of Model A on the MNIST CrossValidation Dataset is: " + str(acc2))
probs = np.array(probs)
if (probs.shape[0] == 1):
probs = np.squeeze(probs, axis=0)
elif (self.model_name == 'cnn' or self.model_name == 'inceptionv3'):
if (self.model_name == 'inceptionv3'):
probs, _, _, acc = self.inception_classifier.get_output(
data, True, output)
else:
probs, _, _, acc = self.CNN_classifier.get_predictions(
data, True, output)
print("Accuracy of Model A on the" + self.dataset_name +
"Training Dataset is: " + str(acc))
probs = np.array(probs)
if (probs.shape[0] == 1):
probs = np.squeeze(probs, axis=0)
elif (self.model_name == 'dt'):
probs2 = self.classifier.predict_model(data)
#print(probs2[0])
probs = convert_to_all_classes_array(probs2,
self.classifier.classes_,
self.output_classes)
if (probs2.shape[-1] == self.output_classes):
assert (probs2.all() == probs.all())
#print("Accuracy of Model A on the current split of the dataset is : ", accuracy_score(output, np.argmax(probs2, axis = -1)))
elif (self.model_name == "ensemble"):
probs = np.array(
self.classifier.predict_ensemble(
data, True, convert_one_hot(output, self.output_classes)))
# actual_probs = np.exp(probs2)/(np.sum(np.exp(probs2), axis = 1))
prob_A_indexes = np.argmax(probs, axis=-1)
print(
"Accuracy of Model A on the current split of the predictions of Model B of the dataset is: ",
accuracy_score(preds, prob_A_indexes))
##print("Classes predicted by the model A: ", np.unique(prob_A_indexes))
##print(probs_train.shape, preds_train.shape, prob_A_indexes.shape)
total_diff = 0.0
if (self.dataset_name == "sentiment_analysis"):
count_equal, count_unequal = 0, 0
for i in range(probs.shape[0]):
if (preds[i] == prob_A_indexes[i]):
count_equal += 1
else:
count_unequal += 1
diff1 = abs(probs[i][preds[i]] - probs_B[i][preds[i]])
val1 = 0.0
if (diff1 != 1.0):
val1 = -1.0 * (math.log2(1.0 - diff1))
else:
max_val = -1.0 * math.ceil(math.log2(entropy_precision))
val1 = max_val
total_diff += val1
total_diff = (total_diff) / (probs.shape[0])
prob_equal = (count_equal * 1.0) / (probs.shape[0])
prob_unequal = (count_unequal * 1.0) / (probs.shape[0])
#total_diff = -1.0 * (prob_equal) * math.log2(prob_equal)
if (prob_equal == 0 or prob_unequal == 0):
total_diff = 0
else:
#total_diff = -1.0 * math.log2(prob_equal)
total_diff = (-1.0 * (prob_equal) * math.log2(prob_equal)) + (
-1.0 * (prob_unequal) * (math.log2(prob_unequal)))
else:
list_diff = []
for i in range(probs.shape[0]):
if (prob_A_indexes[i] != preds[i]):
list_diff.append([i, prob_A_indexes[i], preds[i]])
#print(probs[i][prob_A_indexes[i]])
val = (abs(probs[i][prob_A_indexes[i]] - probs[i][preds[i]]))
if (val <= 0):
total_diff += 0.0
else:
total_diff += -1.0 * (math.log2(val))
total_diff = (total_diff) / (probs.shape[0])
if (len(list_diff) == 0):
print(
"For Model A " + str(self.model_name) +
" and Model B, the final predictions on this split of the dataset are same"
)
else:
None
#print("The no of different values are: " + str(len(list_diff)))
# + " and list is: "
#print(list_diff)
return total_diff
def dump_data(self, data_file, **kwargs):
#if(!append):
# f_handle = open(data_file, 'wb')
arguments = ''
for key, val in kwargs.items():
arguments += str(val['name'])
arguments += "=kwargs['" + str(key) + "']['val'], "
arguments = arguments.strip()
arguments = arguments[:-1]
##print(arguments)
#args = dict(e.split('=') for e in arguments.split(', '))
#eval("np.savez_compressed(self.dump_file_handle, " + arguments + ")")
eval("np.savez_compressed(data_file, " + arguments + ")")
class ModelA(object):
def __init__(self, sess, epochs, batch_size, dataset_name, learning_rate, model_name, is_binarized, is_resized, is_grayscale, feature_size = None):
self.sess = sess
self.dataset_name = dataset_name
self.learning_rate = learning_rate
self.model_name = model_name
self.no_epochs = epochs
self.batch_size = batch_size
self.is_resized = is_resized
self.is_grayscale = is_grayscale
# self.classifier = SGDClassifier(loss = "hinge", penalty = "l2", max_iter = 5)
self.is_binarized = is_binarized
self.no_features = feature_size
if (dataset_name == 'mnist'):
if(self.is_resized):
self.input_image_size = (10, 10, 1)
else:
self.input_image_size = (28, 28, 1)
if (is_binarized):
self.output_classes = 2
else:
self.output_classes = 10
# self.data_X, self.data_Y, self.test_X, self.test_Y = load_mnist_dataset(self.is_binarized)
elif (dataset_name == 'cifar-10'):
if(self.is_grayscale):
self.input_image_size = (28, 28, 1) if (self.is_resized) else (32, 32, 1)
else:
self.input_image_size = (32, 32, 3)
self.output_classes = 10
elif (dataset_name == 'fashion_mnist'):
self.input_image_size = (10, 10, 1) if (self.is_resized) else (28, 28, 1)
self.output_classes = 2 if (self.is_binarized) else 10
elif(dataset_name == 'stanford40'):
self.input_image_size = (200, 200, 3)
self.output_classes = 40
elif(dataset_name == 'sentiment_analysis'):
self.input_image_size = (self.no_features, )
self.output_classes = 2
else:
print("Not Implemented yet")
if (self.model_name == "svm"):
self.classifier = svm.SVC(C=1, kernel='rbf', gamma='auto', probability=True, random_state=0)
#self.classifier = SVC(kernel = 'rbf', gamma = 'auto', C = 1, probability = True, verbose = True, random_state = 0)
self.classifier = fit_model_to_initial_dataset(self.dataset_name, self.classifier, "svm", self.is_resized, self.is_grayscale)
elif (self.model_name == "ann"):
input_size = np.prod(self.input_image_size)
self.hidden_layers = [256]
self.NN = Neural_Network(self.sess, self.no_epochs, self.batch_size, self.learning_rate, input_size,
self.output_classes, self.hidden_layers)
self.NN.create_tf_model("ModelA")
#self.fit_model_to_initial_dataset()
elif(self.model_name == "dt"):
self.classifier = DecisionTree('gini', 'best', None, 5)
print("Gini_Best_5")
self.classifier = fit_model_to_initial_dataset(self.dataset_name, self.classifier, "dt", self.is_resized, self.is_grayscale)
elif(self.model_name == "ensemble"):
input_size = np.prod(self.input_image_size)
self.classifier = ensemble_model(input_size, self.no_epochs, self.batch_size, self.output_classes, self.learning_rate, self.sess, "")
ensemble_list = [("svm", ), ("lr", ), ("dt", "gini", "best", None, 10)]
self.classifier.initialize_ensemble(ensemble_list, self.dataset_name, self.is_resized, self.is_grayscale)
print(ensemble_list)
#("svm", ), ("ann", [512, 128]), ("knn", ),
#
#self.classifier.initialize_ensemble([("svm", ), ("dt", "gini", "best", None, 10)], self.dataset_name, self.is_resized, self.is_grayscale)
#print("Ensemble being used: ann of 64 neurons\n")
elif(self.model_name == "cnn"):
self.CNN_classifier = CNN(self.input_image_size, self.no_epochs, self.batch_size, self.output_classes, self.learning_rate)
self.CNN_classifier.initialize_model()
elif(self.model_name == "inceptionv3"):
self.inception_classifier = Inceptionv3(self.output_classes, self.batch_size, self.no_epochs, self.input_image_size)
self.inception_classifier.initialize_model()
def set_dataset(self, train_X, train_Y, test_X, test_Y, cv_X, cv_Y):
self.data_X, self.data_Y, self.test_X, self.test_Y, self.cross_validation_X, self.cross_validation_Y = train_X, train_Y, test_X, test_Y, cv_X, cv_Y
def calculate_interpretability(self, predictions_model_B_train, predictions_model_B_cross_validation, predictions_model_B_test, dump_bool):
if(self.model_name =='ann' or self.model_name == 'svm' or self.model_name == 'dt' or self.model_name == "ensemble"):
self.data_X = self.data_X.reshape(self.data_X.shape[0], -1)
self.cross_validation_X = self.cross_validation_X.reshape(self.cross_validation_X.shape[0], -1)
self.test_X = self.test_X.reshape(self.test_X.shape[0], -1)
print(self.data_X.shape, self.cross_validation_X.shape)
if(dump_bool):
#self.dump_file_handle = open('dumped_data.npz', 'ab')
input_data_dict = {}
preds_B_dict = {}
if(self.model_name == "svm"):
#input_data_dict['val'] = self.data_X[:10000] CHANGE
#preds_B_dict['val'] = predictions_model_B_train[:10000] CHANGE
input_data_dict['val'] = self.data_X
preds_B_dict['val'] = predictions_model_B_train
else:
input_data_dict['val'] = self.data_X
preds_B_dict['val'] = predictions_model_B_train
input_data_dict['name'] = 'train_data_X'
preds_B_dict['name'] = 'model_B_predictions'
self.dump_data('model_B.npz', input_data_dict = input_data_dict, preds_B_dict = preds_B_dict)
##print("The size of the final dataset used for interpretation is: ", self.data_X.shape)
initial_entropy_train = self.calculate_entropy(predictions_model_B_train, dump_bool, 'initial', 0)
initial_entropy_cv = self.calculate_entropy(predictions_model_B_cross_validation, False, 'initial', 1)
initial_entropy_test = self.calculate_entropy(predictions_model_B_test, False, 'initial', 2)
#initial_entropy_train, initial_entropy_cv, initial_entropy_test = None, None, None
print(initial_entropy_train, initial_entropy_cv, initial_entropy_test)
self.initialize_model(predictions_model_B_train, predictions_model_B_cross_validation)
final_entropy_train = self.calculate_entropy(predictions_model_B_train, dump_bool, 'final', 0)
final_entropy_cv = self.calculate_entropy(predictions_model_B_cross_validation, False, 'final', 1)
final_entropy_test = self.calculate_entropy(predictions_model_B_test, False, 'final', 2)
print("The initial entropy on Train Dataset is :", initial_entropy_train)
print("The initial entropy on Cross Validation Dataset is :", initial_entropy_cv)
print("The final entropy on Train Dataset is : ", final_entropy_train)
print("The final entropy on Cross Validation Dataset is : ", final_entropy_cv)
if(initial_entropy_train == 0.0):
interpret_train = 1.0
else:
interpret_train = (initial_entropy_train - final_entropy_train) / initial_entropy_train
if(initial_entropy_cv == 0.0):
interpret_cv = 1.0
else:
interpret_cv = (initial_entropy_cv - final_entropy_cv) / initial_entropy_cv
if(initial_entropy_test == 0.0):
interpret_test = 1.0
else:
interpret_test = (initial_entropy_test - final_entropy_test)/initial_entropy_test
#self.dump_file_handle.close()
print(interpret_train, interpret_cv, interpret_test)
#re-initialize the classifier to original state for SVM and Decision Tree.
#This is necessary to ensure proper cross-validation.
if(self.model_name == 'svm' or self.model_name == 'dt'):
self.classifier = fit_model_to_initial_dataset(self.dataset_name, self.classifier, self.model_name, self.is_resized, self.is_grayscale)
elif(self.model_name == "ensemble"):
self.classifier.fit_model(self.is_resized, self.is_grayscale)
return interpret_train, interpret_cv, interpret_test
def initialize_model(self, predictions_model_B_train, predictions_model_B_cv):
if (self.model_name == 'svm'):
# self.classifier = SGDClassifier(loss = "hinge")
print(self.data_X.shape, predictions_model_B_train.shape)
#self.classifier.fit(self.data_X[:10000], predictions_model_B[:10000]) CHANGE
#TODO::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::ADD CROSS VALIDATION HERE
self.classifier.fit(self.data_X, predictions_model_B_train)
print("Fitted the SVM to the Predictions of Model B")
elif (self.model_name == 'ann'):
self.data_X = self.data_X.reshape(self.data_X.shape[0], -1)
print(self.data_X.shape, predictions_model_B_train.shape, self.cross_validation_X.shape)
self.NN.train_model(self.data_X, predictions_model_B_train, self.cross_validation_X, predictions_model_B_cv)
print("Trained the Neural Network Model A on Predictions of Model B")
elif(self.model_name == 'dt'):
self.data_X = self.data_X.reshape(self.data_X.shape[0], -1)
self.cross_validation_X = self.cross_validation_X.reshape(self.cross_validation_X.shape[0], -1)
#preds2 = np.argmax(self.classifier.predict_model(self.data_X), axis = -1)
#print("Final Accuracy of Model A on MNIST Train Dataset before interpretation is :" + str(metrics.accuracy_score(self.data_Y, preds2)))
self.classifier.train_model(self.data_X, predictions_model_B_train, self.cross_validation_X, predictions_model_B_cv)
#print("Fitted the Decision Tree to the predictions of model B")
#preds = np.argmax(self.classifier.predict_model(self.cross_validation_X), axis = -1)
#print("Final Accuracy of Model A on MNIST CV Dataset is :" + str(metrics.accuracy_score(self.cross_validation_Y, preds)))
#preds2 = np.argmax(self.classifier.predict_model(self.data_X), axis = -1)
#print("Final Accuracy of Model A on MNIST CV Dataset is :" + str(metrics.accuracy_score(self.data_Y, preds2)))
elif(self.model_name == "ensemble"): #cannot have CNN here, as the data_X is being flattened out here, for use by other models
self.data_X = self.data_X.reshape(self.data_X.shape[0], -1)
self.cross_validation_X = self.cross_validation_X.reshape(self.cross_validation_X.shape[0], -1)
self.classifier.train_ensemble(self.data_X, predictions_model_B_train, self.cross_validation_X, predictions_model_B_cv)
elif(self.model_name == "cnn"):
self.CNN_classifier.train_model(self.data_X, predictions_model_B_train, self.cross_validation_X, predictions_model_B_cv)
elif(self.model_name == "inceptionv3"):
self.inception_classifier.train_model(self.data_X, predictions_model_B_train, self.cross_validation_X, predictions_model_B_cv)
else:
print("Not implemented yet")
def calculate_entropy(self, preds, dump_bool, name, split):
# prob = np.array(self.classifier.decision_function(self.data_X))
# prob_B_indexes = np.argmax(predictions_model_B, axis = -1)
if(split == 0):
data = self.data_X
output = self.data_Y
#print("Train split")
elif(split == 1):
data = self.cross_validation_X
output = self.cross_validation_Y
#print("Cross Validation split ")
elif(split == 2):
data = self.test_X
output = self.test_Y
#print("Test split")
else:
#print("Invalid Split Value")
return None
if (self.model_name == 'svm'):
#probs_train = np.array(self.classifier.predict_proba(self.data_X[:10000])) CHANGE
probs2 = np.array(self.classifier.predict_proba(data))
probs = convert_to_all_classes_array(probs2, self.classifier.classes_, self.output_classes)
if(probs2.shape[-1] == self.output_classes):
assert(probs2.all() == probs.all())
print("Accuracy of Model A on the current split of the dataset is : ", accuracy_score(output, np.argmax(probs2, axis = -1)))
elif (self.model_name == 'ann'):
probs, _, _, acc = self.NN.get_predictions(data, True, convert_one_hot(output, self.output_classes)) # These are 1X50000 arrays
print(probs.shape)
print("Accuracy of Model A on the" + self.dataset_name + "Training Dataset is: " + str(acc))
#print("Accuracy of Model A on the MNIST CrossValidation Dataset is: " + str(acc2))
probs = np.array(probs)
if (probs.shape[0] == 1):
probs = np.squeeze(probs, axis=0)
elif(self.model_name == 'cnn' or self.model_name == 'inceptionv3'):
if(self.model_name == 'inceptionv3'):
probs, _, _, acc = self.inception_classifier.get_output(data, True, output)
else:
probs, _, _, acc = self.CNN_classifier.get_predictions(data, True, output)
print("Accuracy of Model A on the" + self.dataset_name + "Training Dataset is: " + str(acc))
probs = np.array(probs)
if (probs.shape[0] == 1):
probs = np.squeeze(probs, axis=0)
elif(self.model_name == 'dt'):
probs2 = self.classifier.predict_model(data)
#print(probs2[0])
probs = convert_to_all_classes_array(probs2, self.classifier.classes_, self.output_classes)
if(probs2.shape[-1] == self.output_classes):
assert(probs2.all() == probs.all())
#print("Accuracy of Model A on the current split of the dataset is : ", accuracy_score(output, np.argmax(probs2, axis = -1)))
elif(self.model_name == "ensemble"):
probs = np.array(self.classifier.predict_ensemble(data, True, convert_one_hot(output, self.output_classes)))
# actual_probs = np.exp(probs2)/(np.sum(np.exp(probs2), axis = 1))
prob_A_indexes = np.argmax(probs, axis=-1)
##print("Classes predicted by the model A: ", np.unique(prob_A_indexes))
if(dump_bool):
name = name + '_model_A_predictions'
dict1 = {}
dict1['val'] = prob_A_indexes
dict1['name'] = name
self.dump_data(name + '.npz', preds_A_dict = dict1)
##print(probs_train.shape, preds_train.shape, prob_A_indexes.shape)
total_diff = 0.0
list_diff = []
for i in range(probs.shape[0]):
if (prob_A_indexes[i] != preds[i]):
list_diff.append([i, prob_A_indexes[i], preds[i]])
#print(probs[i][prob_A_indexes[i]])
val = (abs(probs[i][prob_A_indexes[i]] - probs[i][preds[i]]))
if(val <= 0):
total_diff += 0.0
else:
total_diff += -1.0 * (math.log2(val))
total_diff = (total_diff) / (probs.shape[0])
if (len(list_diff) == 0):
print("For Model A " + str(
self.model_name) + " and Model B as ANN, the final predictions on this split of the dataset are same")
else:
None
#print("The no of different values are: " + str(len(list_diff)))
# + " and list is: "
#print(list_diff)
return total_diff
def dump_data(self, data_file, **kwargs):
#if(!append):
# f_handle = open(data_file, 'wb')
arguments = ''
for key, val in kwargs.items():
arguments += str(val['name'])
arguments += "=kwargs['" + str(key) + "']['val'], "
arguments = arguments.strip()
arguments = arguments[:-1]
##print(arguments)
#args = dict(e.split('=') for e in arguments.split(', '))
#eval("np.savez_compressed(self.dump_file_handle, " + arguments + ")")
eval("np.savez_compressed(data_file, " + arguments + ")")