def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
# train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/l1_model')
l2_model = tf.keras.models.load_model('models/l2_model')
l3_model = tf.keras.models.load_model('models/l3_model')
# l4_model = tf.keras.models.load_model('models/l4_model')
all_model_accuracies = []
all_model_times = []
all_model_simple_percents = []
all_model_complex_percents = []
for i in range(5):
print("Run l1 l2 l3... #" + str(i))
all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations(
l1_model, l2_model, l3_model, x_test, y_test)
all_model_accuracies.append(accuracies)
all_model_times.append(times)
all_model_simple_percents.append(simple_percents)
all_model_complex_percents.append(complex_percents)
print("All Conf Values:", all_conf_values)
print("L1 L2 L3 Accuracies:", np.mean(all_model_accuracies, axis=0))
print("L1 L2 L3 Times:", np.mean(all_model_times, axis=0))
print("L1 L2 L3 Simple Percents:",
np.mean(all_model_simple_percents, axis=0))
print("L1 L2 L3 Complex Percents:",
np.mean(all_model_complex_percents, axis=0))
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
num_classes = 10
#y_train = tf.keras.utils.to_categorical(y_train, num_classes)
train_and_save_models(x_train, y_train)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
#train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/l1_model')
l2_model = tf.keras.models.load_model('models/l2_model')
l3_model = tf.keras.models.load_model('models/l3_model')
# l4_model = tf.keras.models.load_model('models/l4_model')
p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations(
l1_model, l2_model, x_test, y_test)
print("L1 L2 values:", p_yy, p_ny, p_yn, p_nn)
print(yn_values)
print(ny_values)
p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations(
l1_model, l3_model, x_test, y_test)
print("L1 L3 values:", p_yy, p_ny, p_yn, p_nn)
print(yn_values)
print(ny_values)
p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations(
l2_model, l3_model, x_test, y_test)
print("L2 L3 values:", p_yy, p_ny, p_yn, p_nn)
print(yn_values)
print(ny_values)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
# train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/l1_model')
l2_model = tf.keras.models.load_model('models/l2_model')
l3_model = tf.keras.models.load_model('models/l3_model')
# l4_model = tf.keras.models.load_model('models/l4_model')
# before_time = time.time()
# accuracy = l0_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l1_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l2_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l3_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l4_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
all_model_accuracies = []
all_model_times = []
all_model_simple_percents = []
all_model_complex_percents = []
all_model_thresholds = []
for i in range(1):
print("Run l1 l2... #" + str(i))
all_conf_values, accuracies, times, simple_percents, thresholds = run_combinations(
l2_model, l3_model, x_test, y_test)
all_model_accuracies.append(accuracies)
all_model_times.append(times)
all_model_simple_percents.append(simple_percents)
all_model_thresholds.append(thresholds)
print("All Conf Values:", all_conf_values)
print("L1 L2 Accuracies:", np.mean(all_model_accuracies, axis=0))
print("L1 L2 Times:", np.mean(all_model_times, axis=0))
print("L1 L2 Simple Percents:", np.mean(all_model_simple_percents, axis=0))
print("L1 L2 Thresholds:", np.mean(all_model_thresholds, axis=0))
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
y_test = tf.squeeze(y_test)
#train_and_save_models(x_train, y_train)
print("Loading models...")
l1_model = tf.keras.models.load_model('models/mnist/l1_model')
l2_model = tf.keras.models.load_model('models/mnist/l2_model')
l3_model = tf.keras.models.load_model('models/mnist/l3_model')
l4_model = tf.keras.models.load_model('models/mnist/l4_model')
l5_model = tf.keras.models.load_model('models/mnist/l5_model')
l6_model = tf.keras.models.load_model('models/mnist/l6_model')
l7_model = tf.keras.models.load_model('models/mnist/l7_model')
l8_model = tf.keras.models.load_model('models/mnist/l8_model')
l9_model = tf.keras.models.load_model('models/mnist/l9_model')
l10_model = tf.keras.models.load_model('models/mnist/l10_model')
# Get dictionary of counts of each class in y_test
y_test_np = y_test.numpy()
#unique, counts = np.unique(y_test.numpy(), return_counts=True)
count_dicts = []
# Set up accuracy grid
accuracies = np.zeros((10, 10))
# Iterate over all models and get their predicted outputs
models = [l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model]
for i in range(10):
model = models[i]
model_probs = model.predict(x_test)
model_preds = np.argmax(model_probs, axis=1)
unique, counts = np.unique(model_preds, return_counts=True)
count_dicts.append(dict(zip(unique, counts)))
# Iterate over all 10 classes
for j in range(10):
# Compute the number of times where the prediction matches the test output for that class
class_count = len(np.where((model_preds == j) & (y_test_np == j))[0])
accuracies[i][j] = class_count / count_dicts[i][j]
print(accuracies)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
#train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/l1_model')
l2_model = tf.keras.models.load_model('models/l2_model')
l3_model = tf.keras.models.load_model('models/l3_model')
# l4_model = tf.keras.models.load_model('models/l4_model')
p_yy, p_yn, p_ny, p_nn, yn_sub, ny_sub, nn_sub = run_combinations(
l1_model, l2_model, x_test, y_test)
print("L1 L2 values:", p_yy, p_ny, p_yn, p_nn)
#print("YY:", yy_values)
print("YN:", yn_sub / p_yn)
print("NY:", ny_sub / p_ny)
print("NN:", nn_sub / p_nn)
#print("L1: ", l1_values[:1000])
p_yy, p_yn, p_ny, p_nn, yn_sub, ny_sub, nn_sub = run_combinations(
l1_model, l3_model, x_test, y_test)
print("L1 L3 values:", p_yy, p_ny, p_yn, p_nn)
#print("YY:", yy_values)
print("YN:", yn_sub / p_yn)
print("NY:", ny_sub / p_ny)
print("NN:", nn_sub / p_nn)
#print("L1: ", l1_values[:1000])
p_yy, p_yn, p_ny, p_nn, yn_sub, ny_sub, nn_sub = run_combinations(
l2_model, l3_model, x_test, y_test)
print("L2 L3 values:", p_yy, p_ny, p_yn, p_nn)
#print("YY:", yy_values)
print("YN:", yn_sub / p_yn)
print("NY:", ny_sub / p_ny)
print("NN:", nn_sub / p_nn)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
y_train2 = tf.keras.utils.to_categorical(y_train, 10)
#y_test = tf.squeeze(y_test)
print("Loading models...")
l1_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l1_model, epochs=5, verbose=True)
l1_model._estimator_type = "classifier"
l2_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l2_model, epochs=5, verbose=True)
l2_model._estimator_type = "classifier"
l3_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l3_model, epochs=5, verbose=True)
l3_model._estimator_type = "classifier"
l4_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l4_model, epochs=5, verbose=True)
l4_model._estimator_type = "classifier"
ensemble1 = StackingClassifier(estimators=[('l2', l2_model),
('l3', l3_model),
('l4', l4_model)],
final_estimator=RandomForestClassifier(
random_state=42, n_estimators=10))
ensemble2 = StackingClassifier(
estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)],
final_estimator=RandomForestClassifier(random_state=42))
ensemble3 = StackingClassifier(estimators=[('l2', l2_model),
('l3', l3_model),
('l4', l4_model)],
final_estimator=RandomForestClassifier(
random_state=42, n_estimators=500))
ensemble4 = StackingClassifier(
estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)],
final_estimator=HistGradientBoostingClassifier(random_state=42,
max_iter=10))
ensemble5 = StackingClassifier(
estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)],
final_estimator=HistGradientBoostingClassifier(random_state=42))
ensemble1.fit(x_train, y_train)
#print("L123 Done")
ensemble2.fit(x_train, y_train)
#print("L124 Done")
ensemble3.fit(x_train, y_train)
#print("L134 Done")
ensemble4.fit(x_train, y_train)
#print("L234 Done")
ensemble5.fit(x_train, y_train)
#print("L234 Done")
for clf in (ensemble1, ensemble2, ensemble3, ensemble4, ensemble5):
before_time = time.time()
y_pred = clf.predict(x_test)
model_time = time.time() - before_time
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
print("Time: ", model_time)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
#y_test = tf.squeeze(y_test)
print("Loading models...")
l1_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l1_model, epochs=5)
l1_model._estimator_type = "classifier"
l2_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l2_model, epochs=5)
l2_model._estimator_type = "classifier"
l3_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l3_model, epochs=5)
l3_model._estimator_type = "classifier"
l4_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l4_model, epochs=5)
l4_model._estimator_type = "classifier"
ensemble12 = VotingClassifier(estimators=[('l1', l1_model),
('l2', l2_model)],
voting='soft')
ensemble13 = VotingClassifier(estimators=[('l1', l1_model),
('l3', l3_model)],
voting='soft')
ensemble14 = VotingClassifier(estimators=[('l1', l1_model),
('l4', l4_model)],
voting='soft')
ensemble23 = VotingClassifier(estimators=[('l2', l2_model),
('l3', l3_model)],
voting='soft')
ensemble24 = VotingClassifier(estimators=[('l2', l2_model),
('l4', l4_model)],
voting='soft')
ensemble34 = VotingClassifier(estimators=[('l3', l3_model),
('l4', l4_model)],
voting='soft')
ensemble123 = VotingClassifier(estimators=[('l1', l1_model),
('l2', l2_model),
('l3', l3_model)],
voting='soft')
ensemble124 = VotingClassifier(estimators=[('l1', l1_model),
('l2', l2_model),
('l4', l4_model)],
voting='soft')
ensemble134 = VotingClassifier(estimators=[('l1', l1_model),
('l3', l3_model),
('l4', l4_model)],
voting='soft')
ensemble234 = VotingClassifier(estimators=[('l2', l2_model),
('l3', l3_model),
('l4', l4_model)],
voting='soft')
ensemble1234 = VotingClassifier(estimators=[('l1', l1_model),
('l2', l2_model),
('l3', l3_model),
('l4', l4_model)],
voting='soft')
l1_model.fit(x_train, y_train)
l2_model.fit(x_train, y_train)
l3_model.fit(x_train, y_train)
l4_model.fit(x_train, y_train)
ensemble12.fit(x_train, y_train)
ensemble13.fit(x_train, y_train)
ensemble14.fit(x_train, y_train)
ensemble23.fit(x_train, y_train)
ensemble24.fit(x_train, y_train)
ensemble34.fit(x_train, y_train)
ensemble123.fit(x_train, y_train)
ensemble124.fit(x_train, y_train)
ensemble134.fit(x_train, y_train)
ensemble234.fit(x_train, y_train)
ensemble1234.fit(x_train, y_train)
for clf in (l1_model, l2_model, l3_model, l4_model, ensemble12, ensemble13,
ensemble14, ensemble23, ensemble24, ensemble34, ensemble123,
ensemble124, ensemble134, ensemble234, ensemble1234):
before_time = time.time()
y_pred = clf.predict(x_test)
model_time = time.time() - before_time
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
print("Time: ", model_time)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
y_train2 = tf.keras.utils.to_categorical(y_train, 10)
#y_test = tf.squeeze(y_test)
print("Loading models...")
l1_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l1_model, epochs=5)
l1_model._estimator_type = "classifier"
l2_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l2_model, epochs=5)
l2_model._estimator_type = "classifier"
l3_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l3_model, epochs=5)
l3_model._estimator_type = "classifier"
l4_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l4_model, epochs=5)
l4_model._estimator_type = "classifier"
l1_model.fit(x_train, y_train2)
l2_model.fit(x_train, y_train2)
l3_model.fit(x_train, y_train2)
l4_model.fit(x_train, y_train2)
accuracy_scores = []
for clf in (l1_model, l2_model, l3_model, l4_model):
before_time = time.time()
y_pred = clf.predict(x_test)
model_time = time.time() - before_time
accuracy = accuracy_score(y_test, y_pred)
accuracy_scores.append(accuracy)
print(clf.__class__.__name__, accuracy)
print("Time: ", model_time)
indices = [0, 1]
ensemble12 = VotingClassifier(
estimators=[('l1', l1_model), ('l2', l2_model)],
voting='soft',
weights=[accuracy_scores[i] for i in indices])
indices = [0, 2]
ensemble13 = VotingClassifier(
estimators=[('l1', l1_model), ('l3', l3_model)],
voting='soft',
weights=[accuracy_scores[i] for i in indices])
indices = [0, 3]
ensemble14 = VotingClassifier(
estimators=[('l1', l1_model), ('l4', l4_model)],
voting='soft',
weights=[accuracy_scores[i] for i in indices])
indices = [1, 2]
ensemble23 = VotingClassifier(
estimators=[('l2', l2_model), ('l3', l3_model)],
voting='soft',
weights=[accuracy_scores[i] for i in indices])
indices = [1, 3]
ensemble24 = VotingClassifier(
estimators=[('l2', l2_model), ('l4', l4_model)],
voting='soft',
weights=[accuracy_scores[i] for i in indices])
indices = [2, 3]
ensemble34 = VotingClassifier(
estimators=[('l3', l3_model), ('l4', l4_model)],
voting='soft',
weights=[accuracy_scores[i] for i in indices])
indices = [0, 1, 2]
ensemble123 = VotingClassifier(
estimators=[('l1', l1_model), ('l2', l2_model), ('l3', l3_model)],
voting='soft',
weights=[accuracy_scores[i] for i in indices])
indices = [0, 1, 3]
ensemble124 = VotingClassifier(
estimators=[('l1', l1_model), ('l2', l2_model), ('l4', l4_model)],
voting='soft',
weights=[accuracy_scores[i] for i in indices])
indices = [0, 2, 3]
ensemble134 = VotingClassifier(
estimators=[('l1', l1_model), ('l3', l3_model), ('l4', l4_model)],
voting='soft',
weights=[accuracy_scores[i] for i in indices])
indices = [1, 2, 3]
ensemble234 = VotingClassifier(
estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)],
voting='soft',
weights=[accuracy_scores[i] for i in indices])
ensemble1234 = VotingClassifier(estimators=[('l1', l1_model),
('l2', l2_model),
('l3', l3_model),
('l4', l4_model)],
voting='soft',
weights=accuracy_scores)
ensemble12.fit(x_train, y_train)
ensemble13.fit(x_train, y_train)
ensemble14.fit(x_train, y_train)
ensemble23.fit(x_train, y_train)
ensemble24.fit(x_train, y_train)
ensemble34.fit(x_train, y_train)
ensemble123.fit(x_train, y_train)
ensemble124.fit(x_train, y_train)
ensemble134.fit(x_train, y_train)
ensemble234.fit(x_train, y_train)
ensemble1234.fit(x_train, y_train)
for clf in (l1_model, l2_model, l3_model, l4_model, ensemble12, ensemble13,
ensemble14, ensemble23, ensemble24, ensemble34, ensemble123,
ensemble124, ensemble134, ensemble234, ensemble1234):
before_time = time.time()
y_pred = clf.predict(x_test)
model_time = time.time() - before_time
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
print("Time: ", model_time)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
#train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/l1_model')
l2_model = tf.keras.models.load_model('models/l2_model')
l3_model = tf.keras.models.load_model('models/l3_model')
# l4_model = tf.keras.models.load_model('models/l4_model')
l1_time = 0
l2_time = 0
l3_time = 0
for i in range(5):
before_time = time.time()
accuracy = l1_model.predict(x_test)
l1_time += time.time() - before_time
print(l1_time)
before_time = time.time()
accuracy = l2_model.predict(x_test)
l2_time += time.time() - before_time
before_time = time.time()
accuracy = l3_model.predict(x_test)
l3_time += time.time() - before_time
print("L1 Time:", l1_time / 5)
print("L2 Time:", l2_time / 5)
print("L3 Time:", l3_time / 5)
l1_l2_accuracies = []
l1_l3_accuracies = []
l2_l3_accuracies = []
l1_l2_times = []
l1_l3_times = []
l2_l3_times = []
l1_l2_percents = []
l1_l3_percents = []
l2_l3_percents = []
for i in range(5):
print("Run l1 l2... #" + str(i))
accuracies, times, simplePercents = run_combinations(l1_model, l2_model, x_test, y_test)
l1_l2_accuracies.append(accuracies)
l1_l2_times.append(times)
l1_l2_percents.append(simplePercents)
print("Run l1 l3... #" + str(i))
accuracies, times, simplePercents = run_combinations(l1_model, l3_model, x_test, y_test)
l1_l3_accuracies.append(accuracies)
l1_l3_times.append(times)
l1_l3_percents.append(simplePercents)
print("Run l2 l3... #" + str(i))
accuracies, times, simplePercents = run_combinations(l2_model, l3_model, x_test, y_test)
l2_l3_accuracies.append(accuracies)
l2_l3_times.append(times)
l2_l3_percents.append(simplePercents)
print("L1 L2 Accuracies:", np.mean(l1_l2_accuracies, axis=0))
print("L1 L2 Times:", np.mean(l1_l2_times, axis=0))
print("L1 L2 Simple Percents", np.mean(l1_l2_percents, axis=0))
print("L1 L3 Accuracies:", np.mean(l1_l3_accuracies, axis=0))
print("L1 L3 Times:", np.mean(l1_l3_times, axis=0))
print("L1 L3 Simple Percents", np.mean(l1_l3_percents, axis=0))
print("L2 L3 Accuracies:", np.mean(l2_l3_accuracies, axis=0))
print("L2 L3 Times:", np.mean(l2_l3_times, axis=0))
print("L2 L3 Simple Percents", np.mean(l2_l3_percents, axis=0))
def main():
run_greedy_walk = True
run_naive = False
# ======================================================================
# Preparing data and trained models
print('Loading data...')
# Get MNIST data
x_train, y_train, x_test, y_test = helpers.get_mnist_data()
# For testing model to sort by complexity
x_test_1 = x_test[:3333]
y_test_1 = y_test[:3333]
# For finding optimal confidence value
x_test_2 = x_test[3334:6666]
y_test_2 = y_test[3334:6666]
# For validating optimal confidence value
x_test_3 = x_test[6667:10000]
y_test_3 = y_test[6667:10000]
print('Loading models...')
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/l1_model')
l2_model = tf.keras.models.load_model('models/l2_model')
l3_model = tf.keras.models.load_model('models/l3_model')
print('Storing models in Graph object...')
# Load saved models and store as Graph.Model object (with corresponding complexity)
models_list = []
for i in range(1, 4):
current_model = tf.keras.models.load_model('models/l' + str(i) + '_model')
current_model_object = Graph.Model(current_model)
before_time = time.time()
results = current_model.evaluate(x_test_1, y_test_1, verbose=0)
after_time = time.time()
current_model_object.accuracy = results[1]
current_model_object.time = after_time - before_time
current_model_object.complexity_index = i - 1
print('complexity_index:', current_model_object.complexity_index, \
'accuracy:', current_model_object.accuracy, 'time: ', current_model_object.time)
models_list.append(current_model_object)
# Create graph object and add models sorted based on complexity
graph = Graph.Graph(models_list)
# ======================================================================
# Running Greedy Walk:
if run_greedy_walk:
print('Running Greedy Walk...')
greedy_output = []
# for current_model_search_time in np.flip(np.arange(0.2, 0.5, 0.05)):
print("=========================================================")
current_model_search_time = 0.45
print("Greedy:", current_model_search_time)
output, search_time, best_node_accuracy = greedy_walk(graph, 20, \
current_model_search_time, x_test_2, y_test_2, x_test_3, y_test_3)
if output == None:
print('Greedy walk did not find any viable pairing')
greedy_output.append(None)
else:
print('Greedy walk found', output.complexities, 'in', search_time, 'seconds')
greedy_output.append((output.complexities, search_time, best_node_accuracy, output.optimal_confidence_value))
print(greedy_output)
# ======================================================================
# Running Naive:
if run_naive:
naive_output = []
# for current_model_search_time in np.arange(0.2, 0.5, 0.05):
current_model_search_time = 0.35
print("=========================================================")
print("Naive:", current_model_search_time)
output, search_time, best_node_accuracy = naive_search(models_list, current_model_search_time,\
x_test_2, y_test_2, x_test_3, y_test_3)
if output == None:
print('Naive did not any find viable pairing')
naive_output.append(None)
else:
print('Naive found', output.complexities, 'in', search_time, 'seconds')
naive_output.append((output.complexities, search_time, best_node_accuracy, output.optimal_confidence_value))
print(naive_output)