def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
num_classes = 10
#y_train = tf.keras.utils.to_categorical(y_train, num_classes)
train_and_save_models(x_train, y_train)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
# train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
# l4_model = tf.keras.models.load_model('models/l4_model')
all_model_accuracies = []
all_model_times = []
all_model_simple_percents = []
all_model_complex_percents = []
for i in range(2):
print("Run l1 l2 l3... #" + str(i))
all_p_values, accuracies, times, simple_percents, complex_percents = run_combinations(
l1_model, l2_model, l3_model, x_test, y_test)
all_model_accuracies.append(accuracies)
all_model_times.append(times)
all_model_simple_percents.append(simple_percents)
all_model_complex_percents.append(complex_percents)
print("All P Values:", all_p_values)
print("L1 L2 L3 Accuracies:", np.mean(all_model_accuracies, axis=0))
print("L1 L2 L3 Times:", np.mean(all_model_times, axis=0))
print("L1 L2 L3 Simple Percents:",
np.mean(all_model_simple_percents, axis=0))
print("L1 L2 L3 Complex Percents:",
np.mean(all_model_complex_percents, axis=0))
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
#train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
#l4_model = tf.keras.models.load_model('models/cifar/l4_model')
p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations(
l1_model, l2_model, x_test, y_test)
print("L1 L2 values:", p_yy, p_ny, p_yn, p_nn)
#print(yn_values)
#print(ny_values[:1000])
p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations(
l1_model, l3_model, x_test, y_test)
print("L1 L3 values:", p_yy, p_ny, p_yn, p_nn)
#print(yn_values)
#print(ny_values[:1000])
#p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations(l1_model, l4_model, x_test, y_test)
#print("L1 L4 values:", p_yy, p_ny, p_yn, p_nn)
#print(yn_values)
#print(ny_values[:1000])
p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations(
l2_model, l3_model, x_test, y_test)
print("L2 L3 values:", p_yy, p_ny, p_yn, p_nn)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
# train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
l4_model = tf.keras.models.load_model('models/cifar/l4_model')
l5_model = tf.keras.models.load_model('models/cifar/l5_model')
l6_model = tf.keras.models.load_model('models/cifar/l6_model')
l7_model = tf.keras.models.load_model('models/cifar/l7_model')
l8_model = tf.keras.models.load_model('models/cifar/l8_model')
l_accuracies = []
l_times = []
l_simple_percents = []
l_complex_percents = []
for i in range(2):
print("Run l6 l7 l8... #" + str(i))
all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations(l5_model, l7_model, l8_model, x_test, y_test)
l_accuracies.append(accuracies)
l_times.append(times)
l_simple_percents.append(simple_percents)
l_complex_percents.append(complex_percents)
print("Accuracies:", np.mean(l_accuracies, axis=0))
print("Times:", np.mean(l_times, axis=0))
print("Simple Percents:", np.mean(l_simple_percents, axis=0))
print("Complex Percents:", np.mean(l_complex_percents, axis=0))
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
# train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
# l4_model = tf.keras.models.load_model('models/l4_model')
# before_time = time.time()
# accuracy = l0_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l1_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l2_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l3_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l4_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
all_model_accuracies = []
all_model_times = []
all_model_simple_percents = []
all_model_complex_percents = []
all_model_thresholds = []
for i in range(1):
print("Run l1 l2... #" + str(i))
all_conf_values, accuracies, times, simple_percents, thresholds = run_combinations(l1_model, l3_model, x_test, y_test)
all_model_accuracies.append(accuracies)
all_model_times.append(times)
all_model_simple_percents.append(simple_percents)
all_model_thresholds.append(thresholds)
print("All Conf Values:", all_conf_values)
print("L1 L2 Accuracies:", np.mean(all_model_accuracies, axis=0))
print("L1 L2 Times:", np.mean(all_model_times, axis=0))
print("L1 L2 Simple Percents:", np.mean(all_model_simple_percents, axis=0))
print("L1 L2 Thresholds:", np.mean(all_model_thresholds, axis=0))
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
#train_and_save_models(x_train, y_train)
print("Loading models...")
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
l4_model = tf.keras.models.load_model('models/cifar/l4_model')
l5_model = tf.keras.models.load_model('models/cifar/l5_model')
l6_model = tf.keras.models.load_model('models/cifar/l6_model')
l7_model = tf.keras.models.load_model('models/cifar/l7_model')
l8_model = tf.keras.models.load_model('models/cifar/l8_model')
l9_model = tf.keras.models.load_model('models/cifar/l9_model')
l10_model = tf.keras.models.load_model('models/cifar/l10_model')
# Get dictionary of counts of each class in y_test
y_test_np = y_test.numpy()
unique, counts = np.unique(y_test.numpy(), return_counts=True)
count_dict = dict(zip(unique, counts))
# Set up accuracy grid
accuracies = np.zeros((10, 10))
# Iterate over all models and get their predicted outputs
models = [l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model]
models_preds = []
for i in range(10):
model = models[i]
model_probs = model.predict(x_test)
model_preds = np.argmax(model_probs, axis=1)
models_preds.append(model_preds)
for i in range(10):
model1 = models_preds[i]
for j in range(10):
model2 = models_preds[j]
# Compute the number of times where the two models match predictions
model_count = np.count_nonzero(model1 == model2)
accuracies[i][j] = model_count / 10000
print(accuracies)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
#train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
# l1_model = tf.keras.models.load_model('models/cifar/l1_model')
# l2_model = tf.keras.models.load_model('models/cifar/l2_model')
# l3_model = tf.keras.models.load_model('models/cifar/l3_model')
# l4_model = tf.keras.models.load_model('models/cifar/l4_model')
l7_model = tf.keras.models.load_model('models/cifar/l7_model')
l8_model = tf.keras.models.load_model('models/cifar/l8_model')
l7_times = []
l8_times = []
# for i in range(5):
# before_time = time.time()
# accuracy = l7_model.predict(x_test)
# #print("L1 Accuracy: ", l1_model.evaluate(x_test, y_test, verbose=0)[1])
# l7_times.append(time.time() - before_time)
# before_time = time.time()
# accuracy = l8_model.predict(x_test)
# #print("L2 Accuracy: ", l2_model.evaluate(x_test, y_test, verbose=0)[1])
# l8_times.append(time.time() - before_time)
# print(l7_times)
# print(l8_times)
# print("L7 Time:", np.mean(l7_times[1:], axis=0))
# print("L8 Time:", np.mean(l8_times[1:], axis=0))
l7_l8_accuracies = []
l7_l8_times = []
l7_l8_percents = []
for i in range(3):
print("Run l7 l8... #" + str(i))
accuracies, times, simplePercents = run_combinations(
l7_model, l8_model, x_test, y_test)
l7_l8_accuracies.append(accuracies)
l7_l8_times.append(times)
l7_l8_percents.append(simplePercents)
print("L7 L8 Accuracies:", np.mean(l7_l8_accuracies, axis=0))
print("L7 L8 Times:", np.mean(l7_l8_times, axis=0))
print("L7 L8 Simple Percents", np.mean(l7_l8_percents, axis=0))
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
#train_and_save_models(x_train, y_train)
print("Loading models...")
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
l4_model = tf.keras.models.load_model('models/cifar/l4_model')
l5_model = tf.keras.models.load_model('models/cifar/l5_model')
l6_model = tf.keras.models.load_model('models/cifar/l6_model')
l7_model = tf.keras.models.load_model('models/cifar/l7_model')
l8_model = tf.keras.models.load_model('models/cifar/l8_model')
l9_model = tf.keras.models.load_model('models/cifar/l9_model')
l10_model = tf.keras.models.load_model('models/cifar/l10_model')
# Get dictionary of counts of each class in y_test
y_test_np = y_test.numpy()
unique, counts = np.unique(y_test.numpy(), return_counts=True)
count_dict = dict(zip(unique, counts))
# Set up accuracy grid
accuracies = np.zeros((10, 10))
# Iterate over all models and get their predicted outputs
models = [
l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model,
l8_model, l9_model, l10_model
]
for i in range(10):
model = models[i]
model_probs = model.predict(x_test)
model_preds = np.argmax(model_probs, axis=1)
# Iterate over all 10 classes
for j in range(10):
# Compute the number of times where the prediction matches the test output for that class
class_count = len(
np.where((model_preds == j) & (y_test_np == j))[0])
accuracies[i][j] = class_count / count_dict[j]
print(accuracies)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
print("Loading models...")
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
l4_model = tf.keras.models.load_model('models/cifar/l4_model')
l5_model = tf.keras.models.load_model('models/cifar/l5_model')
l6_model = tf.keras.models.load_model('models/cifar/l6_model')
l7_model = tf.keras.models.load_model('models/cifar/l7_model')
l8_model = tf.keras.models.load_model('models/cifar/l8_model')
l9_model = tf.keras.models.load_model('models/cifar/l9_model')
l10_model = tf.keras.models.load_model('models/cifar/l10_model')
models = [
l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model,
l8_model, l9_model, l10_model
]
num_classes = 10
accuracies = compute_class_matrix_A(models, num_classes, x_test, y_test)
best_models = np.argmax(accuracies, axis=0)
# Classify the test data into buckets based off of their true class
buckets = [[] for i in range(num_classes)]
for i in range(x_test.shape[0]):
class_peek = y_test[i]
buckets[class_peek].append(x_test[i])
# Loop through each of the class buckets and run the corresponding model, before adding up the correct predictions
total_correct = 0
for i in range(num_classes):
bucket_inputs = np.array(buckets[i])
print(best_models[i])
model = models[best_models[i]]
probs = model.predict(bucket_inputs)
preds = np.argmax(probs, axis=1)
total_correct += np.sum(preds == i)
print("Final accuracy: ", total_correct / x_test.shape[0])
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
#train_and_save_models(x_train, y_train)
print("Loading models...")
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
l4_model = tf.keras.models.load_model('models/cifar/l4_model')
l5_model = tf.keras.models.load_model('models/cifar/l5_model')
l6_model = tf.keras.models.load_model('models/cifar/l6_model')
l7_model = tf.keras.models.load_model('models/cifar/l7_model')
l8_model = tf.keras.models.load_model('models/cifar/l8_model')
l9_model = tf.keras.models.load_model('models/cifar/l9_model')
l10_model = tf.keras.models.load_model('models/cifar/l10_model')
models = [l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model]
num_models = len(models)
num_samples = x_test.shape[0]
output_size = 10
# Pre-compute model outputs
model_outputs = np.zeros((num_models, num_samples, output_size))
for i in range(num_models):
print("Loading model " + str(i + 1) + " outputs...")
model = models[i]
model_probs = model.predict(x_test)
model_outputs[i] = model_probs
# Compute all combinations
for i in range(num_models):
for j in range(num_models):
if i == j:
continue
else:
print("Running L" + str(i + 1) + " L" + str(j + 1) + "...")
accuracies = run_combinations(model_outputs[i], model_outputs[j], y_test)
print("Accuracies:", accuracies)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
print("Loading models...")
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
l4_model = tf.keras.models.load_model('models/cifar/l4_model')
l5_model = tf.keras.models.load_model('models/cifar/l5_model')
l6_model = tf.keras.models.load_model('models/cifar/l6_model')
l7_model = tf.keras.models.load_model('models/cifar/l7_model')
l8_model = tf.keras.models.load_model('models/cifar/l8_model')
l9_model = tf.keras.models.load_model('models/cifar/l9_model')
l10_model = tf.keras.models.load_model('models/cifar/l10_model')
models = [l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model]
num_classes = 10
#unweighted_optimize(models, num_classes, x_test, y_test)
weighted_optimize(models, num_classes, x_test, y_test, weight_type="B")
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
print("Loading models...")
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
l4_model = tf.keras.models.load_model('models/cifar/l4_model')
l5_model = tf.keras.models.load_model('models/cifar/l5_model')
l6_model = tf.keras.models.load_model('models/cifar/l6_model')
l7_model = tf.keras.models.load_model('models/cifar/l7_model')
l8_model = tf.keras.models.load_model('models/cifar/l8_model')
l9_model = tf.keras.models.load_model('models/cifar/l9_model')
l10_model = tf.keras.models.load_model('models/cifar/l10_model')
models = [l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model]
num_classes = 10
thresholds = [0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]
weight_types = ["A", "B", "C", "O", "U"]
run_experiment(models, num_classes, x_test, y_test, thresholds=thresholds, weight_types=weight_types, iterations=10)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
# train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
l4_model = tf.keras.models.load_model('models/cifar/l4_model')
# before_time = time.time()
# accuracy = l0_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l1_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l2_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l3_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
# before_time = time.time()
# accuracy = l4_model.evaluate(x_test, y_test)
# print("Time", time.time() - before_time)
l123_accuracies = []
l124_accuracies = []
l134_accuracies = []
l234_accuracies = []
l123_times = []
l124_times = []
l134_times = []
l234_times = []
l123_simple_percents = []
l124_simple_percents = []
l134_simple_percents = []
l234_simple_percents = []
l123_complex_percents = []
l124_complex_percents = []
l134_complex_percents = []
l234_complex_percents = []
for i in range(2):
print("Run l1 l2 l3... #" + str(i))
all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations(
l1_model, l2_model, l3_model, x_test, y_test)
l123_accuracies.append(accuracies)
l123_times.append(times)
l123_simple_percents.append(simple_percents)
l123_complex_percents.append(complex_percents)
print("Run l1 l2 l4... #" + str(i))
all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations(
l1_model, l2_model, l4_model, x_test, y_test)
l124_accuracies.append(accuracies)
l124_times.append(times)
l124_simple_percents.append(simple_percents)
l124_complex_percents.append(complex_percents)
print("Run l1 l3 l4... #" + str(i))
all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations(
l1_model, l3_model, l4_model, x_test, y_test)
l134_accuracies.append(accuracies)
l134_times.append(times)
l134_simple_percents.append(simple_percents)
l134_complex_percents.append(complex_percents)
print("Run l2 l3 l4... #" + str(i))
all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations(
l2_model, l3_model, l4_model, x_test, y_test)
l234_accuracies.append(accuracies)
l234_times.append(times)
l234_simple_percents.append(simple_percents)
l234_complex_percents.append(complex_percents)
print("L1 L2 L3 Accuracies:", np.mean(l123_accuracies, axis=0))
print("L1 L2 L3 Times:", np.mean(l123_times, axis=0))
print("L1 L2 L3 Simple Percents:", np.mean(l123_simple_percents, axis=0))
print("L1 L2 L3 Complex Percents:", np.mean(l123_complex_percents, axis=0))
print("L1 L2 L4 Accuracies:", np.mean(l124_accuracies, axis=0))
print("L1 L2 L4 Times:", np.mean(l124_times, axis=0))
print("L1 L2 L4 Simple Percents:", np.mean(l124_simple_percents, axis=0))
print("L1 L2 L4 Complex Percents:", np.mean(l124_complex_percents, axis=0))
print("L1 L3 L4 Accuracies:", np.mean(l134_accuracies, axis=0))
print("L1 L3 L4 Times:", np.mean(l134_times, axis=0))
print("L1 L3 L4 Simple Percents:", np.mean(l134_simple_percents, axis=0))
print("L1 L3 L4 Complex Percents:", np.mean(l134_complex_percents, axis=0))
print("L2 L3 L4 Accuracies:", np.mean(l234_accuracies, axis=0))
print("L2 L3 L4 Times:", np.mean(l234_times, axis=0))
print("L2 L3 L4 Simple Percents:", np.mean(l234_simple_percents, axis=0))
print("L2 L3 L4 Complex Percents:", np.mean(l234_complex_percents, axis=0))
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
#train_and_save_models(x_train, y_train)
print("Loading models...")
# l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
l4_model = tf.keras.models.load_model('models/cifar/l4_model')
# l1_time = 0
# l2_time = 0
# l3_time = 0
# l4_time = 0
# for i in range(5):
# before_time = time.time()
# accuracy = l1_model.predict(x_test)
# #print("L1 Accuracy: ", l1_model.evaluate(x_test, y_test, verbose=0)[1])
# l1_time += time.time() - before_time
# before_time = time.time()
# accuracy = l2_model.predict(x_test)
# #print("L2 Accuracy: ", l2_model.evaluate(x_test, y_test, verbose=0)[1])
# l2_time += time.time() - before_time
# before_time = time.time()
# accuracy = l3_model.predict(x_test)
# #print("L3 Accuracy: ", l3_model.evaluate(x_test, y_test, verbose=0)[1])
# l3_time += time.time() - before_time
# before_time = time.time()
# accuracy = l4_model.predict(x_test)
# #print("L4 Accuracy: ", l4_model.evaluate(x_test, y_test, verbose=0)[1])
# l4_time += time.time() - before_time
# print("L1 Time:", l1_time / 5)
# print("L2 Time:", l2_time / 5)
# print("L3 Time:", l3_time / 5)
l1_l2_accuracies = []
l1_l3_accuracies = []
l2_l3_accuracies = []
l1_l2_times = []
l1_l3_times = []
l2_l3_times = []
l1_l2_percents = []
l1_l3_percents = []
l2_l3_percents = []
l1_l4_accuracies = []
l2_l4_accuracies = []
l3_l4_accuracies = []
l1_l4_times = []
l2_l4_times = []
l3_l4_times = []
l1_l4_percents = []
l2_l4_percents = []
l3_l4_percents = []
for i in range(3):
print("Run l1 l2... #" + str(i))
accuracies, times, simplePercents = run_combinations(
l1_model, l2_model, x_test, y_test)
l1_l2_accuracies.append(accuracies)
l1_l2_times.append(times)
l1_l2_percents.append(simplePercents)
print("Run l1 l3... #" + str(i))
accuracies, times, simplePercents = run_combinations(
l1_model, l3_model, x_test, y_test)
l1_l3_accuracies.append(accuracies)
l1_l3_times.append(times)
l1_l3_percents.append(simplePercents)
print("Run l2 l3... #" + str(i))
accuracies, times, simplePercents = run_combinations(
l2_model, l3_model, x_test, y_test)
l2_l3_accuracies.append(accuracies)
l2_l3_times.append(times)
l2_l3_percents.append(simplePercents)
print("Run l1 l4... #" + str(i))
accuracies, times, simplePercents = run_combinations(
l1_model, l4_model, x_test, y_test)
l1_l4_accuracies.append(accuracies)
l1_l4_times.append(times)
l1_l4_percents.append(simplePercents)
print("Run l2 l4... #" + str(i))
accuracies, times, simplePercents = run_combinations(
l2_model, l4_model, x_test, y_test)
l2_l4_accuracies.append(accuracies)
l2_l4_times.append(times)
l2_l4_percents.append(simplePercents)
print("Run l3 l4... #" + str(i))
accuracies, times, simplePercents = run_combinations(
l3_model, l4_model, x_test, y_test)
l3_l4_accuracies.append(accuracies)
l3_l4_times.append(times)
l3_l4_percents.append(simplePercents)
print("L1 L2 Accuracies:", np.mean(l1_l2_accuracies, axis=0))
print("L1 L2 Times:", np.mean(l1_l2_times, axis=0))
print("L1 L2 Simple Percents", np.mean(l1_l2_percents, axis=0))
print("L1 L3 Accuracies:", np.mean(l1_l3_accuracies, axis=0))
print("L1 L3 Times:", np.mean(l1_l3_times, axis=0))
print("L1 L3 Simple Percents", np.mean(l1_l3_percents, axis=0))
print("L2 L3 Accuracies:", np.mean(l2_l3_accuracies, axis=0))
print("L2 L3 Times:", np.mean(l2_l3_times, axis=0))
print("L2 L3 Simple Percents", np.mean(l2_l3_percents, axis=0))
print("L1 L4 Accuracies:", np.mean(l1_l4_accuracies, axis=0))
print("L1 L4 Times:", np.mean(l1_l4_times, axis=0))
print("L1 L4 Simple Percents", np.mean(l1_l4_percents, axis=0))
print("L2 L4 Accuracies:", np.mean(l2_l4_accuracies, axis=0))
print("L2 L4 Times:", np.mean(l2_l4_times, axis=0))
print("L2 L4 Simple Percents", np.mean(l2_l4_percents, axis=0))
print("L3 L4 Accuracies:", np.mean(l3_l4_accuracies, axis=0))
print("L3 L4 Times:", np.mean(l3_l4_times, axis=0))
print("L3 L4 Simple Percents", np.mean(l3_l4_percents, axis=0))
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test2 = tf.squeeze(y_test)
#train_and_save_models(x_train, y_train)
print("Loading models...")
#l0_model = tf.keras.models.load_model('models/l0_model')
l1_model = tf.keras.models.load_model('models/cifar/l1_model')
l2_model = tf.keras.models.load_model('models/cifar/l2_model')
l3_model = tf.keras.models.load_model('models/cifar/l3_model')
l4_model = tf.keras.models.load_model('models/cifar/l4_model')
l5_model = tf.keras.models.load_model('models/cifar/l5_model')
l6_model = tf.keras.models.load_model('models/cifar/l6_model')
l7_model = tf.keras.models.load_model('models/cifar/l7_model')
l8_model = tf.keras.models.load_model('models/cifar/l8_model')
l9_model = tf.keras.models.load_model('models/cifar/l9_model')
l10_model = tf.keras.models.load_model('models/cifar/l10_model')
# l1_model = tf.keras.models.load_model('models/mnist/l1_model')
# l2_model = tf.keras.models.load_model('models/mnist/l2_model')
# l3_model = tf.keras.models.load_model('models/mnist/l3_model')
# l4_model = tf.keras.models.load_model('models/mnist/l4_model')
# l5_model = tf.keras.models.load_model('models/mnist/l5_model')
# l6_model = tf.keras.models.load_model('models/mnist/l6_model')
# l7_model = tf.keras.models.load_model('models/mnist/l7_model')
# l8_model = tf.keras.models.load_model('models/mnist/l8_model')
# l9_model = tf.keras.models.load_model('models/mnist/l9_model')
# l10_model = tf.keras.models.load_model('models/mnist/l10_model')
l1_time = []
l2_time = []
l3_time = []
l4_time = []
l5_time = []
l6_time = []
l7_time = []
l8_time = []
l9_time = []
l10_time = []
for i in range(5):
before_time = time.time()
#accuracy = l1_model.predict(x_test)
print("L1 Accuracy: ",
l1_model.evaluate(x_test, y_test2, verbose=0)[1])
l1_time.append(time.time() - before_time)
before_time = time.time()
#accuracy = l2_model.predict(x_test)
print("L2 Accuracy: ",
l2_model.evaluate(x_test, y_test2, verbose=0)[1])
l2_time.append(time.time() - before_time)
before_time = time.time()
#accuracy = l3_model.predict(x_test)
print("L3 Accuracy: ",
l3_model.evaluate(x_test, y_test2, verbose=0)[1])
l3_time.append(time.time() - before_time)
before_time = time.time()
#accuracy = l4_model.predict(x_test)
print("L4 Accuracy: ",
l4_model.evaluate(x_test, y_test2, verbose=0)[1])
l4_time.append(time.time() - before_time)
before_time = time.time()
#accuracy = l3_model.predict(x_test)
print("L5 Accuracy: ",
l5_model.evaluate(x_test, y_test2, verbose=0)[1])
l5_time.append(time.time() - before_time)
before_time = time.time()
#accuracy = l4_model.predict(x_test)
print("L6 Accuracy: ",
l6_model.evaluate(x_test, y_test2, verbose=0)[1])
l6_time.append(time.time() - before_time)
before_time = time.time()
#accuracy = l3_model.predict(x_test)
print("L7 Accuracy: ",
l7_model.evaluate(x_test, y_test2, verbose=0)[1])
l7_time.append(time.time() - before_time)
before_time = time.time()
#accuracy = l4_model.predict(x_test)
print("L8 Accuracy: ",
l8_model.evaluate(x_test, y_test2, verbose=0)[1])
l8_time.append(time.time() - before_time)
before_time = time.time()
# #accuracy = l3_model.predict(x_test)
print("L9 Accuracy: ",
l9_model.evaluate(x_test, y_test2, verbose=0)[1])
l9_time.append(time.time() - before_time)
before_time = time.time()
# #accuracy = l4_model.predict(x_test)
print("L10 Accuracy: ",
l10_model.evaluate(x_test, y_test2, verbose=0)[1])
l10_time.append(time.time() - before_time)
print("L1 Time:", np.mean(l1_time[1:], axis=0), l1_time)
print("L2 Time:", np.mean(l2_time[1:], axis=0), l2_time)
print("L3 Time:", np.mean(l3_time[1:], axis=0), l3_time)
print("L4 Time:", np.mean(l4_time[1:], axis=0), l4_time)
print("L5 Time:", np.mean(l5_time[1:], axis=0), l5_time)
print("L6 Time:", np.mean(l6_time[1:], axis=0), l6_time)
print("L7 Time:", np.mean(l3_time[1:], axis=0), l7_time)
print("L8 Time:", np.mean(l4_time[1:], axis=0), l8_time)
print("L9 Time:", np.mean(l5_time[1:], axis=0), l9_time)
print("L10 Time:", np.mean(l6_time[1:], axis=0), l10_time)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_test = tf.squeeze(y_test)
print("Loading models...")
l1_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l1_model, epochs=10, verbose=True)
l1_model._estimator_type = "classifier"
l2_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l2_model, epochs=10, verbose=True)
l2_model._estimator_type = "classifier"
l3_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l3_model, epochs=10, verbose=True)
l3_model._estimator_type = "classifier"
l4_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l4_model, epochs=10, verbose=True)
l4_model._estimator_type = "classifier"
l5_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l5_model, epochs=10, verbose=True)
l5_model._estimator_type = "classifier"
l6_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l6_model, epochs=10, verbose=True)
l6_model._estimator_type = "classifier"
l7_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l7_model, epochs=10, verbose=True)
l7_model._estimator_type = "classifier"
l8_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l8_model, epochs=10, verbose=True)
l8_model._estimator_type = "classifier"
l9_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l9_model, epochs=10, verbose=True)
l9_model._estimator_type = "classifier"
l10_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l10_model, epochs=10, verbose=True)
l10_model._estimator_type = "classifier"
# ensemble12 = VotingClassifier(estimators=[('l1', l1_model),
# ('l2', l2_model)],
# voting='soft', verbose=False)
# ensemble13 = VotingClassifier(estimators=[('l1', l1_model),
# ('l3', l3_model)],
# voting='soft', verbose=False)
# ensemble14 = VotingClassifier(estimators=[('l1', l1_model),
# ('l4', l4_model)],
# voting='soft', verbose=False)
# ensemble23 = VotingClassifier(estimators=[('l2', l2_model),
# ('l3', l3_model)],
# voting='soft', verbose=False)
# ensemble24 = VotingClassifier(estimators=[('l2', l2_model),
# ('l4', l4_model)],
# voting='soft', verbose=False)
# ensemble34 = VotingClassifier(estimators=[('l3', l3_model),
# ('l4', l4_model)],
# voting='soft', verbose=False)
# ensemble123 = VotingClassifier(estimators=[('l1', l1_model),
# ('l2', l2_model),
# ('l3', l3_model)],
# voting='soft', verbose=False)
# ensemble124 = VotingClassifier(estimators=[('l1', l1_model),
# ('l2', l2_model),
# ('l4', l4_model)],
# voting='soft', verbose=False)
# ensemble134 = VotingClassifier(estimators=[('l1', l1_model),
# ('l3', l3_model),
# ('l4', l4_model)],
# voting='soft', verbose=False)
# ensemble234 = VotingClassifier(estimators=[('l2', l2_model),
# ('l3', l3_model),
# ('l4', l4_model)],
# voting='soft', verbose=False)
# ensemble1234 = VotingClassifier(estimators=[('l1', l1_model),
# ('l2', l2_model),
# ('l3', l3_model),
# ('l4', l4_model)],
# voting='soft', verbose=True)
# ensemble5678 = VotingClassifier(estimators=[('l5', l5_model),
# ('l6', l6_model),
# ('l7', l7_model),
# ('l8', l8_model)],
# voting='soft', verbose=True)
ensemble = VotingClassifier(estimators=[('l1', l1_model),
('l2', l2_model),
('l3', l3_model),
('l4', l4_model),
('l5', l5_model),
('l6', l6_model),
('l7', l7_model),
('l8', l8_model),
('l9', l9_model),
('l10', l10_model)],
voting='hard', verbose=True)
# l1_model.fit(x_train, y_train)
# l2_model.fit(x_train, y_train)
# l3_model.fit(x_train, y_train)
# l4_model.fit(x_train, y_train)
# ensemble12.fit(x_train, y_train)
# ensemble13.fit(x_train, y_train)
# ensemble14.fit(x_train, y_train)
# ensemble23.fit(x_train, y_train)
# ensemble24.fit(x_train, y_train)
# ensemble34.fit(x_train, y_train)
# ensemble123.fit(x_train, y_train)
# ensemble124.fit(x_train, y_train)
# ensemble134.fit(x_train, y_train)
# ensemble234.fit(x_train, y_train)
# ensemble1234.fit(x_train, y_train)
# ensemble5678.fit(x_train, y_train)
ensemble.fit(x_train, y_train)
for clf in (ensemble, ensemble):
before_time = time.time()
y_pred = clf.predict(x_test)
model_time = time.time() - before_time
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
print("Time: ", model_time)
def main():
print('Loading data...')
x_train, y_train, x_test, y_test = helpers.get_cifar10_data()
y_train2 = tf.keras.utils.to_categorical(y_train, 10)
y_test = tf.squeeze(y_test)
print("Loading models...")
l1_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l1_model, epochs=10, verbose=True)
l1_model._estimator_type = "classifier"
l2_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l2_model, epochs=10, verbose=True)
l2_model._estimator_type = "classifier"
l3_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l3_model, epochs=10, verbose=True)
l3_model._estimator_type = "classifier"
l4_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l4_model, epochs=10, verbose=True)
l4_model._estimator_type = "classifier"
l5_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l5_model, epochs=10, verbose=True)
l5_model._estimator_type = "classifier"
l6_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l6_model, epochs=10, verbose=True)
l6_model._estimator_type = "classifier"
l7_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l7_model, epochs=10, verbose=True)
l7_model._estimator_type = "classifier"
l8_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l8_model, epochs=10, verbose=True)
l8_model._estimator_type = "classifier"
l9_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l9_model, epochs=10, verbose=True)
l9_model._estimator_type = "classifier"
l10_model = tf.keras.wrappers.scikit_learn.KerasClassifier(
build_fn=models.get_l10_model, epochs=10, verbose=True)
l10_model._estimator_type = "classifier"
l1_model.fit(x_train, y_train2)
l2_model.fit(x_train, y_train2)
l3_model.fit(x_train, y_train2)
l4_model.fit(x_train, y_train2)
l5_model.fit(x_train, y_train)
l6_model.fit(x_train, y_train)
l7_model.fit(x_train, y_train)
l8_model.fit(x_train, y_train)
l9_model.fit(x_train, y_train)
l10_model.fit(x_train, y_train)
accuracy_scores = []
for clf in (l1_model, l2_model, l3_model, l4_model, l5_model, l6_model,
l7_model, l8_model, l9_model, l10_model):
before_time = time.time()
y_pred = clf.predict(x_test)
model_time = time.time() - before_time
accuracy = accuracy_score(y_test, y_pred)
accuracy_scores.append(accuracy)
print(clf.__class__.__name__, accuracy)
print("Time: ", model_time)
# indices = [0, 1]
# ensemble12 = VotingClassifier(estimators=[('l1', l1_model),
# ('l2', l2_model)],
# voting='soft', weights=[accuracy_scores[i] for i in indices])
# indices = [0, 2]
# ensemble13 = VotingClassifier(estimators=[('l1', l1_model),
# ('l3', l3_model)],
# voting='soft', weights=[accuracy_scores[i] for i in indices])
# indices = [0, 3]
# ensemble14 = VotingClassifier(estimators=[('l1', l1_model),
# ('l4', l4_model)],
# voting='soft', weights=[accuracy_scores[i] for i in indices])
# indices = [1, 2]
# ensemble23 = VotingClassifier(estimators=[('l2', l2_model),
# ('l3', l3_model)],
# voting='soft', weights=[accuracy_scores[i] for i in indices])
# indices = [1, 3]
# ensemble24 = VotingClassifier(estimators=[('l2', l2_model),
# ('l4', l4_model)],
# voting='soft', weights=[accuracy_scores[i] for i in indices])
# indices = [2, 3]
# ensemble34 = VotingClassifier(estimators=[('l3', l3_model),
# ('l4', l4_model)],
# voting='soft', weights=[accuracy_scores[i] for i in indices])
# indices = [0, 1, 2]
# ensemble123 = VotingClassifier(estimators=[('l1', l1_model),
# ('l2', l2_model),
# ('l3', l3_model)],
# voting='soft', weights=[accuracy_scores[i] for i in indices])
# indices = [0, 1, 3]
# ensemble124 = VotingClassifier(estimators=[('l1', l1_model),
# ('l2', l2_model),
# ('l4', l4_model)],
# voting='soft', weights=[accuracy_scores[i] for i in indices])
# indices = [0, 2, 3]
# ensemble134 = VotingClassifier(estimators=[('l1', l1_model),
# ('l3', l3_model),
# ('l4', l4_model)],
# voting='soft', weights=[accuracy_scores[i] for i in indices])
# indices = [1, 2, 3]
# ensemble234 = VotingClassifier(estimators=[('l2', l2_model),
# ('l3', l3_model),
# ('l4', l4_model)],
# voting='soft', weights=[accuracy_scores[i] for i in indices])
ensemble = VotingClassifier(estimators=[('l1', l1_model), ('l2', l2_model),
('l3', l3_model), ('l4', l4_model),
('l5', l5_model), ('l6', l6_model),
('l7', l7_model), ('l8', l8_model),
('l9', l9_model),
('l10', l10_model)],
voting='soft',
weights=accuracy_scores,
verbose=True)
# ensemble12.fit(x_train, y_train)
# ensemble13.fit(x_train, y_train)
# ensemble14.fit(x_train, y_train)
# ensemble23.fit(x_train, y_train)
# ensemble24.fit(x_train, y_train)
# ensemble34.fit(x_train, y_train)
# ensemble123.fit(x_train, y_train)
# ensemble124.fit(x_train, y_train)
# ensemble134.fit(x_train, y_train)
# ensemble234.fit(x_train, y_train)
ensemble.fit(x_train, y_train)
for clf in (ensemble, ensemble):
before_time = time.time()
y_pred = clf.predict(x_test)
model_time = time.time() - before_time
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
print("Time: ", model_time)