def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() num_classes = 10 #y_train = tf.keras.utils.to_categorical(y_train, num_classes) train_and_save_models(x_train, y_train)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) # train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') # l4_model = tf.keras.models.load_model('models/l4_model') all_model_accuracies = [] all_model_times = [] all_model_simple_percents = [] all_model_complex_percents = [] for i in range(2): print("Run l1 l2 l3... #" + str(i)) all_p_values, accuracies, times, simple_percents, complex_percents = run_combinations( l1_model, l2_model, l3_model, x_test, y_test) all_model_accuracies.append(accuracies) all_model_times.append(times) all_model_simple_percents.append(simple_percents) all_model_complex_percents.append(complex_percents) print("All P Values:", all_p_values) print("L1 L2 L3 Accuracies:", np.mean(all_model_accuracies, axis=0)) print("L1 L2 L3 Times:", np.mean(all_model_times, axis=0)) print("L1 L2 L3 Simple Percents:", np.mean(all_model_simple_percents, axis=0)) print("L1 L2 L3 Complex Percents:", np.mean(all_model_complex_percents, axis=0))
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) #train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') #l4_model = tf.keras.models.load_model('models/cifar/l4_model') p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations( l1_model, l2_model, x_test, y_test) print("L1 L2 values:", p_yy, p_ny, p_yn, p_nn) #print(yn_values) #print(ny_values[:1000]) p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations( l1_model, l3_model, x_test, y_test) print("L1 L3 values:", p_yy, p_ny, p_yn, p_nn) #print(yn_values) #print(ny_values[:1000]) #p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations(l1_model, l4_model, x_test, y_test) #print("L1 L4 values:", p_yy, p_ny, p_yn, p_nn) #print(yn_values) #print(ny_values[:1000]) p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations( l2_model, l3_model, x_test, y_test) print("L2 L3 values:", p_yy, p_ny, p_yn, p_nn)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) # train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') l4_model = tf.keras.models.load_model('models/cifar/l4_model') l5_model = tf.keras.models.load_model('models/cifar/l5_model') l6_model = tf.keras.models.load_model('models/cifar/l6_model') l7_model = tf.keras.models.load_model('models/cifar/l7_model') l8_model = tf.keras.models.load_model('models/cifar/l8_model') l_accuracies = [] l_times = [] l_simple_percents = [] l_complex_percents = [] for i in range(2): print("Run l6 l7 l8... #" + str(i)) all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations(l5_model, l7_model, l8_model, x_test, y_test) l_accuracies.append(accuracies) l_times.append(times) l_simple_percents.append(simple_percents) l_complex_percents.append(complex_percents) print("Accuracies:", np.mean(l_accuracies, axis=0)) print("Times:", np.mean(l_times, axis=0)) print("Simple Percents:", np.mean(l_simple_percents, axis=0)) print("Complex Percents:", np.mean(l_complex_percents, axis=0))
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() # train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') # l4_model = tf.keras.models.load_model('models/l4_model') # before_time = time.time() # accuracy = l0_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l1_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l2_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l3_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l4_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) all_model_accuracies = [] all_model_times = [] all_model_simple_percents = [] all_model_complex_percents = [] all_model_thresholds = [] for i in range(1): print("Run l1 l2... #" + str(i)) all_conf_values, accuracies, times, simple_percents, thresholds = run_combinations(l1_model, l3_model, x_test, y_test) all_model_accuracies.append(accuracies) all_model_times.append(times) all_model_simple_percents.append(simple_percents) all_model_thresholds.append(thresholds) print("All Conf Values:", all_conf_values) print("L1 L2 Accuracies:", np.mean(all_model_accuracies, axis=0)) print("L1 L2 Times:", np.mean(all_model_times, axis=0)) print("L1 L2 Simple Percents:", np.mean(all_model_simple_percents, axis=0)) print("L1 L2 Thresholds:", np.mean(all_model_thresholds, axis=0))
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) #train_and_save_models(x_train, y_train) print("Loading models...") l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') l4_model = tf.keras.models.load_model('models/cifar/l4_model') l5_model = tf.keras.models.load_model('models/cifar/l5_model') l6_model = tf.keras.models.load_model('models/cifar/l6_model') l7_model = tf.keras.models.load_model('models/cifar/l7_model') l8_model = tf.keras.models.load_model('models/cifar/l8_model') l9_model = tf.keras.models.load_model('models/cifar/l9_model') l10_model = tf.keras.models.load_model('models/cifar/l10_model') # Get dictionary of counts of each class in y_test y_test_np = y_test.numpy() unique, counts = np.unique(y_test.numpy(), return_counts=True) count_dict = dict(zip(unique, counts)) # Set up accuracy grid accuracies = np.zeros((10, 10)) # Iterate over all models and get their predicted outputs models = [l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model] models_preds = [] for i in range(10): model = models[i] model_probs = model.predict(x_test) model_preds = np.argmax(model_probs, axis=1) models_preds.append(model_preds) for i in range(10): model1 = models_preds[i] for j in range(10): model2 = models_preds[j] # Compute the number of times where the two models match predictions model_count = np.count_nonzero(model1 == model2) accuracies[i][j] = model_count / 10000 print(accuracies)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) #train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') # l1_model = tf.keras.models.load_model('models/cifar/l1_model') # l2_model = tf.keras.models.load_model('models/cifar/l2_model') # l3_model = tf.keras.models.load_model('models/cifar/l3_model') # l4_model = tf.keras.models.load_model('models/cifar/l4_model') l7_model = tf.keras.models.load_model('models/cifar/l7_model') l8_model = tf.keras.models.load_model('models/cifar/l8_model') l7_times = [] l8_times = [] # for i in range(5): # before_time = time.time() # accuracy = l7_model.predict(x_test) # #print("L1 Accuracy: ", l1_model.evaluate(x_test, y_test, verbose=0)[1]) # l7_times.append(time.time() - before_time) # before_time = time.time() # accuracy = l8_model.predict(x_test) # #print("L2 Accuracy: ", l2_model.evaluate(x_test, y_test, verbose=0)[1]) # l8_times.append(time.time() - before_time) # print(l7_times) # print(l8_times) # print("L7 Time:", np.mean(l7_times[1:], axis=0)) # print("L8 Time:", np.mean(l8_times[1:], axis=0)) l7_l8_accuracies = [] l7_l8_times = [] l7_l8_percents = [] for i in range(3): print("Run l7 l8... #" + str(i)) accuracies, times, simplePercents = run_combinations( l7_model, l8_model, x_test, y_test) l7_l8_accuracies.append(accuracies) l7_l8_times.append(times) l7_l8_percents.append(simplePercents) print("L7 L8 Accuracies:", np.mean(l7_l8_accuracies, axis=0)) print("L7 L8 Times:", np.mean(l7_l8_times, axis=0)) print("L7 L8 Simple Percents", np.mean(l7_l8_percents, axis=0))
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) #train_and_save_models(x_train, y_train) print("Loading models...") l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') l4_model = tf.keras.models.load_model('models/cifar/l4_model') l5_model = tf.keras.models.load_model('models/cifar/l5_model') l6_model = tf.keras.models.load_model('models/cifar/l6_model') l7_model = tf.keras.models.load_model('models/cifar/l7_model') l8_model = tf.keras.models.load_model('models/cifar/l8_model') l9_model = tf.keras.models.load_model('models/cifar/l9_model') l10_model = tf.keras.models.load_model('models/cifar/l10_model') # Get dictionary of counts of each class in y_test y_test_np = y_test.numpy() unique, counts = np.unique(y_test.numpy(), return_counts=True) count_dict = dict(zip(unique, counts)) # Set up accuracy grid accuracies = np.zeros((10, 10)) # Iterate over all models and get their predicted outputs models = [ l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model ] for i in range(10): model = models[i] model_probs = model.predict(x_test) model_preds = np.argmax(model_probs, axis=1) # Iterate over all 10 classes for j in range(10): # Compute the number of times where the prediction matches the test output for that class class_count = len( np.where((model_preds == j) & (y_test_np == j))[0]) accuracies[i][j] = class_count / count_dict[j] print(accuracies)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) print("Loading models...") l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') l4_model = tf.keras.models.load_model('models/cifar/l4_model') l5_model = tf.keras.models.load_model('models/cifar/l5_model') l6_model = tf.keras.models.load_model('models/cifar/l6_model') l7_model = tf.keras.models.load_model('models/cifar/l7_model') l8_model = tf.keras.models.load_model('models/cifar/l8_model') l9_model = tf.keras.models.load_model('models/cifar/l9_model') l10_model = tf.keras.models.load_model('models/cifar/l10_model') models = [ l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model ] num_classes = 10 accuracies = compute_class_matrix_A(models, num_classes, x_test, y_test) best_models = np.argmax(accuracies, axis=0) # Classify the test data into buckets based off of their true class buckets = [[] for i in range(num_classes)] for i in range(x_test.shape[0]): class_peek = y_test[i] buckets[class_peek].append(x_test[i]) # Loop through each of the class buckets and run the corresponding model, before adding up the correct predictions total_correct = 0 for i in range(num_classes): bucket_inputs = np.array(buckets[i]) print(best_models[i]) model = models[best_models[i]] probs = model.predict(bucket_inputs) preds = np.argmax(probs, axis=1) total_correct += np.sum(preds == i) print("Final accuracy: ", total_correct / x_test.shape[0])
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) #train_and_save_models(x_train, y_train) print("Loading models...") l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') l4_model = tf.keras.models.load_model('models/cifar/l4_model') l5_model = tf.keras.models.load_model('models/cifar/l5_model') l6_model = tf.keras.models.load_model('models/cifar/l6_model') l7_model = tf.keras.models.load_model('models/cifar/l7_model') l8_model = tf.keras.models.load_model('models/cifar/l8_model') l9_model = tf.keras.models.load_model('models/cifar/l9_model') l10_model = tf.keras.models.load_model('models/cifar/l10_model') models = [l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model] num_models = len(models) num_samples = x_test.shape[0] output_size = 10 # Pre-compute model outputs model_outputs = np.zeros((num_models, num_samples, output_size)) for i in range(num_models): print("Loading model " + str(i + 1) + " outputs...") model = models[i] model_probs = model.predict(x_test) model_outputs[i] = model_probs # Compute all combinations for i in range(num_models): for j in range(num_models): if i == j: continue else: print("Running L" + str(i + 1) + " L" + str(j + 1) + "...") accuracies = run_combinations(model_outputs[i], model_outputs[j], y_test) print("Accuracies:", accuracies)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) print("Loading models...") l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') l4_model = tf.keras.models.load_model('models/cifar/l4_model') l5_model = tf.keras.models.load_model('models/cifar/l5_model') l6_model = tf.keras.models.load_model('models/cifar/l6_model') l7_model = tf.keras.models.load_model('models/cifar/l7_model') l8_model = tf.keras.models.load_model('models/cifar/l8_model') l9_model = tf.keras.models.load_model('models/cifar/l9_model') l10_model = tf.keras.models.load_model('models/cifar/l10_model') models = [l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model] num_classes = 10 #unweighted_optimize(models, num_classes, x_test, y_test) weighted_optimize(models, num_classes, x_test, y_test, weight_type="B")
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) print("Loading models...") l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') l4_model = tf.keras.models.load_model('models/cifar/l4_model') l5_model = tf.keras.models.load_model('models/cifar/l5_model') l6_model = tf.keras.models.load_model('models/cifar/l6_model') l7_model = tf.keras.models.load_model('models/cifar/l7_model') l8_model = tf.keras.models.load_model('models/cifar/l8_model') l9_model = tf.keras.models.load_model('models/cifar/l9_model') l10_model = tf.keras.models.load_model('models/cifar/l10_model') models = [l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model] num_classes = 10 thresholds = [0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95] weight_types = ["A", "B", "C", "O", "U"] run_experiment(models, num_classes, x_test, y_test, thresholds=thresholds, weight_types=weight_types, iterations=10)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) # train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') l4_model = tf.keras.models.load_model('models/cifar/l4_model') # before_time = time.time() # accuracy = l0_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l1_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l2_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l3_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l4_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) l123_accuracies = [] l124_accuracies = [] l134_accuracies = [] l234_accuracies = [] l123_times = [] l124_times = [] l134_times = [] l234_times = [] l123_simple_percents = [] l124_simple_percents = [] l134_simple_percents = [] l234_simple_percents = [] l123_complex_percents = [] l124_complex_percents = [] l134_complex_percents = [] l234_complex_percents = [] for i in range(2): print("Run l1 l2 l3... #" + str(i)) all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations( l1_model, l2_model, l3_model, x_test, y_test) l123_accuracies.append(accuracies) l123_times.append(times) l123_simple_percents.append(simple_percents) l123_complex_percents.append(complex_percents) print("Run l1 l2 l4... #" + str(i)) all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations( l1_model, l2_model, l4_model, x_test, y_test) l124_accuracies.append(accuracies) l124_times.append(times) l124_simple_percents.append(simple_percents) l124_complex_percents.append(complex_percents) print("Run l1 l3 l4... #" + str(i)) all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations( l1_model, l3_model, l4_model, x_test, y_test) l134_accuracies.append(accuracies) l134_times.append(times) l134_simple_percents.append(simple_percents) l134_complex_percents.append(complex_percents) print("Run l2 l3 l4... #" + str(i)) all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations( l2_model, l3_model, l4_model, x_test, y_test) l234_accuracies.append(accuracies) l234_times.append(times) l234_simple_percents.append(simple_percents) l234_complex_percents.append(complex_percents) print("L1 L2 L3 Accuracies:", np.mean(l123_accuracies, axis=0)) print("L1 L2 L3 Times:", np.mean(l123_times, axis=0)) print("L1 L2 L3 Simple Percents:", np.mean(l123_simple_percents, axis=0)) print("L1 L2 L3 Complex Percents:", np.mean(l123_complex_percents, axis=0)) print("L1 L2 L4 Accuracies:", np.mean(l124_accuracies, axis=0)) print("L1 L2 L4 Times:", np.mean(l124_times, axis=0)) print("L1 L2 L4 Simple Percents:", np.mean(l124_simple_percents, axis=0)) print("L1 L2 L4 Complex Percents:", np.mean(l124_complex_percents, axis=0)) print("L1 L3 L4 Accuracies:", np.mean(l134_accuracies, axis=0)) print("L1 L3 L4 Times:", np.mean(l134_times, axis=0)) print("L1 L3 L4 Simple Percents:", np.mean(l134_simple_percents, axis=0)) print("L1 L3 L4 Complex Percents:", np.mean(l134_complex_percents, axis=0)) print("L2 L3 L4 Accuracies:", np.mean(l234_accuracies, axis=0)) print("L2 L3 L4 Times:", np.mean(l234_times, axis=0)) print("L2 L3 L4 Simple Percents:", np.mean(l234_simple_percents, axis=0)) print("L2 L3 L4 Complex Percents:", np.mean(l234_complex_percents, axis=0))
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) #train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') l4_model = tf.keras.models.load_model('models/cifar/l4_model') # l1_time = 0 # l2_time = 0 # l3_time = 0 # l4_time = 0 # for i in range(5): # before_time = time.time() # accuracy = l1_model.predict(x_test) # #print("L1 Accuracy: ", l1_model.evaluate(x_test, y_test, verbose=0)[1]) # l1_time += time.time() - before_time # before_time = time.time() # accuracy = l2_model.predict(x_test) # #print("L2 Accuracy: ", l2_model.evaluate(x_test, y_test, verbose=0)[1]) # l2_time += time.time() - before_time # before_time = time.time() # accuracy = l3_model.predict(x_test) # #print("L3 Accuracy: ", l3_model.evaluate(x_test, y_test, verbose=0)[1]) # l3_time += time.time() - before_time # before_time = time.time() # accuracy = l4_model.predict(x_test) # #print("L4 Accuracy: ", l4_model.evaluate(x_test, y_test, verbose=0)[1]) # l4_time += time.time() - before_time # print("L1 Time:", l1_time / 5) # print("L2 Time:", l2_time / 5) # print("L3 Time:", l3_time / 5) l1_l2_accuracies = [] l1_l3_accuracies = [] l2_l3_accuracies = [] l1_l2_times = [] l1_l3_times = [] l2_l3_times = [] l1_l2_percents = [] l1_l3_percents = [] l2_l3_percents = [] l1_l4_accuracies = [] l2_l4_accuracies = [] l3_l4_accuracies = [] l1_l4_times = [] l2_l4_times = [] l3_l4_times = [] l1_l4_percents = [] l2_l4_percents = [] l3_l4_percents = [] for i in range(3): print("Run l1 l2... #" + str(i)) accuracies, times, simplePercents = run_combinations( l1_model, l2_model, x_test, y_test) l1_l2_accuracies.append(accuracies) l1_l2_times.append(times) l1_l2_percents.append(simplePercents) print("Run l1 l3... #" + str(i)) accuracies, times, simplePercents = run_combinations( l1_model, l3_model, x_test, y_test) l1_l3_accuracies.append(accuracies) l1_l3_times.append(times) l1_l3_percents.append(simplePercents) print("Run l2 l3... #" + str(i)) accuracies, times, simplePercents = run_combinations( l2_model, l3_model, x_test, y_test) l2_l3_accuracies.append(accuracies) l2_l3_times.append(times) l2_l3_percents.append(simplePercents) print("Run l1 l4... #" + str(i)) accuracies, times, simplePercents = run_combinations( l1_model, l4_model, x_test, y_test) l1_l4_accuracies.append(accuracies) l1_l4_times.append(times) l1_l4_percents.append(simplePercents) print("Run l2 l4... #" + str(i)) accuracies, times, simplePercents = run_combinations( l2_model, l4_model, x_test, y_test) l2_l4_accuracies.append(accuracies) l2_l4_times.append(times) l2_l4_percents.append(simplePercents) print("Run l3 l4... #" + str(i)) accuracies, times, simplePercents = run_combinations( l3_model, l4_model, x_test, y_test) l3_l4_accuracies.append(accuracies) l3_l4_times.append(times) l3_l4_percents.append(simplePercents) print("L1 L2 Accuracies:", np.mean(l1_l2_accuracies, axis=0)) print("L1 L2 Times:", np.mean(l1_l2_times, axis=0)) print("L1 L2 Simple Percents", np.mean(l1_l2_percents, axis=0)) print("L1 L3 Accuracies:", np.mean(l1_l3_accuracies, axis=0)) print("L1 L3 Times:", np.mean(l1_l3_times, axis=0)) print("L1 L3 Simple Percents", np.mean(l1_l3_percents, axis=0)) print("L2 L3 Accuracies:", np.mean(l2_l3_accuracies, axis=0)) print("L2 L3 Times:", np.mean(l2_l3_times, axis=0)) print("L2 L3 Simple Percents", np.mean(l2_l3_percents, axis=0)) print("L1 L4 Accuracies:", np.mean(l1_l4_accuracies, axis=0)) print("L1 L4 Times:", np.mean(l1_l4_times, axis=0)) print("L1 L4 Simple Percents", np.mean(l1_l4_percents, axis=0)) print("L2 L4 Accuracies:", np.mean(l2_l4_accuracies, axis=0)) print("L2 L4 Times:", np.mean(l2_l4_times, axis=0)) print("L2 L4 Simple Percents", np.mean(l2_l4_percents, axis=0)) print("L3 L4 Accuracies:", np.mean(l3_l4_accuracies, axis=0)) print("L3 L4 Times:", np.mean(l3_l4_times, axis=0)) print("L3 L4 Simple Percents", np.mean(l3_l4_percents, axis=0))
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test2 = tf.squeeze(y_test) #train_and_save_models(x_train, y_train) print("Loading models...") #l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/cifar/l1_model') l2_model = tf.keras.models.load_model('models/cifar/l2_model') l3_model = tf.keras.models.load_model('models/cifar/l3_model') l4_model = tf.keras.models.load_model('models/cifar/l4_model') l5_model = tf.keras.models.load_model('models/cifar/l5_model') l6_model = tf.keras.models.load_model('models/cifar/l6_model') l7_model = tf.keras.models.load_model('models/cifar/l7_model') l8_model = tf.keras.models.load_model('models/cifar/l8_model') l9_model = tf.keras.models.load_model('models/cifar/l9_model') l10_model = tf.keras.models.load_model('models/cifar/l10_model') # l1_model = tf.keras.models.load_model('models/mnist/l1_model') # l2_model = tf.keras.models.load_model('models/mnist/l2_model') # l3_model = tf.keras.models.load_model('models/mnist/l3_model') # l4_model = tf.keras.models.load_model('models/mnist/l4_model') # l5_model = tf.keras.models.load_model('models/mnist/l5_model') # l6_model = tf.keras.models.load_model('models/mnist/l6_model') # l7_model = tf.keras.models.load_model('models/mnist/l7_model') # l8_model = tf.keras.models.load_model('models/mnist/l8_model') # l9_model = tf.keras.models.load_model('models/mnist/l9_model') # l10_model = tf.keras.models.load_model('models/mnist/l10_model') l1_time = [] l2_time = [] l3_time = [] l4_time = [] l5_time = [] l6_time = [] l7_time = [] l8_time = [] l9_time = [] l10_time = [] for i in range(5): before_time = time.time() #accuracy = l1_model.predict(x_test) print("L1 Accuracy: ", l1_model.evaluate(x_test, y_test2, verbose=0)[1]) l1_time.append(time.time() - before_time) before_time = time.time() #accuracy = l2_model.predict(x_test) print("L2 Accuracy: ", l2_model.evaluate(x_test, y_test2, verbose=0)[1]) l2_time.append(time.time() - before_time) before_time = time.time() #accuracy = l3_model.predict(x_test) print("L3 Accuracy: ", l3_model.evaluate(x_test, y_test2, verbose=0)[1]) l3_time.append(time.time() - before_time) before_time = time.time() #accuracy = l4_model.predict(x_test) print("L4 Accuracy: ", l4_model.evaluate(x_test, y_test2, verbose=0)[1]) l4_time.append(time.time() - before_time) before_time = time.time() #accuracy = l3_model.predict(x_test) print("L5 Accuracy: ", l5_model.evaluate(x_test, y_test2, verbose=0)[1]) l5_time.append(time.time() - before_time) before_time = time.time() #accuracy = l4_model.predict(x_test) print("L6 Accuracy: ", l6_model.evaluate(x_test, y_test2, verbose=0)[1]) l6_time.append(time.time() - before_time) before_time = time.time() #accuracy = l3_model.predict(x_test) print("L7 Accuracy: ", l7_model.evaluate(x_test, y_test2, verbose=0)[1]) l7_time.append(time.time() - before_time) before_time = time.time() #accuracy = l4_model.predict(x_test) print("L8 Accuracy: ", l8_model.evaluate(x_test, y_test2, verbose=0)[1]) l8_time.append(time.time() - before_time) before_time = time.time() # #accuracy = l3_model.predict(x_test) print("L9 Accuracy: ", l9_model.evaluate(x_test, y_test2, verbose=0)[1]) l9_time.append(time.time() - before_time) before_time = time.time() # #accuracy = l4_model.predict(x_test) print("L10 Accuracy: ", l10_model.evaluate(x_test, y_test2, verbose=0)[1]) l10_time.append(time.time() - before_time) print("L1 Time:", np.mean(l1_time[1:], axis=0), l1_time) print("L2 Time:", np.mean(l2_time[1:], axis=0), l2_time) print("L3 Time:", np.mean(l3_time[1:], axis=0), l3_time) print("L4 Time:", np.mean(l4_time[1:], axis=0), l4_time) print("L5 Time:", np.mean(l5_time[1:], axis=0), l5_time) print("L6 Time:", np.mean(l6_time[1:], axis=0), l6_time) print("L7 Time:", np.mean(l3_time[1:], axis=0), l7_time) print("L8 Time:", np.mean(l4_time[1:], axis=0), l8_time) print("L9 Time:", np.mean(l5_time[1:], axis=0), l9_time) print("L10 Time:", np.mean(l6_time[1:], axis=0), l10_time)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_test = tf.squeeze(y_test) print("Loading models...") l1_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l1_model, epochs=10, verbose=True) l1_model._estimator_type = "classifier" l2_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l2_model, epochs=10, verbose=True) l2_model._estimator_type = "classifier" l3_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l3_model, epochs=10, verbose=True) l3_model._estimator_type = "classifier" l4_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l4_model, epochs=10, verbose=True) l4_model._estimator_type = "classifier" l5_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l5_model, epochs=10, verbose=True) l5_model._estimator_type = "classifier" l6_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l6_model, epochs=10, verbose=True) l6_model._estimator_type = "classifier" l7_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l7_model, epochs=10, verbose=True) l7_model._estimator_type = "classifier" l8_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l8_model, epochs=10, verbose=True) l8_model._estimator_type = "classifier" l9_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l9_model, epochs=10, verbose=True) l9_model._estimator_type = "classifier" l10_model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=models.get_l10_model, epochs=10, verbose=True) l10_model._estimator_type = "classifier" # ensemble12 = VotingClassifier(estimators=[('l1', l1_model), # ('l2', l2_model)], # voting='soft', verbose=False) # ensemble13 = VotingClassifier(estimators=[('l1', l1_model), # ('l3', l3_model)], # voting='soft', verbose=False) # ensemble14 = VotingClassifier(estimators=[('l1', l1_model), # ('l4', l4_model)], # voting='soft', verbose=False) # ensemble23 = VotingClassifier(estimators=[('l2', l2_model), # ('l3', l3_model)], # voting='soft', verbose=False) # ensemble24 = VotingClassifier(estimators=[('l2', l2_model), # ('l4', l4_model)], # voting='soft', verbose=False) # ensemble34 = VotingClassifier(estimators=[('l3', l3_model), # ('l4', l4_model)], # voting='soft', verbose=False) # ensemble123 = VotingClassifier(estimators=[('l1', l1_model), # ('l2', l2_model), # ('l3', l3_model)], # voting='soft', verbose=False) # ensemble124 = VotingClassifier(estimators=[('l1', l1_model), # ('l2', l2_model), # ('l4', l4_model)], # voting='soft', verbose=False) # ensemble134 = VotingClassifier(estimators=[('l1', l1_model), # ('l3', l3_model), # ('l4', l4_model)], # voting='soft', verbose=False) # ensemble234 = VotingClassifier(estimators=[('l2', l2_model), # ('l3', l3_model), # ('l4', l4_model)], # voting='soft', verbose=False) # ensemble1234 = VotingClassifier(estimators=[('l1', l1_model), # ('l2', l2_model), # ('l3', l3_model), # ('l4', l4_model)], # voting='soft', verbose=True) # ensemble5678 = VotingClassifier(estimators=[('l5', l5_model), # ('l6', l6_model), # ('l7', l7_model), # ('l8', l8_model)], # voting='soft', verbose=True) ensemble = VotingClassifier(estimators=[('l1', l1_model), ('l2', l2_model), ('l3', l3_model), ('l4', l4_model), ('l5', l5_model), ('l6', l6_model), ('l7', l7_model), ('l8', l8_model), ('l9', l9_model), ('l10', l10_model)], voting='hard', verbose=True) # l1_model.fit(x_train, y_train) # l2_model.fit(x_train, y_train) # l3_model.fit(x_train, y_train) # l4_model.fit(x_train, y_train) # ensemble12.fit(x_train, y_train) # ensemble13.fit(x_train, y_train) # ensemble14.fit(x_train, y_train) # ensemble23.fit(x_train, y_train) # ensemble24.fit(x_train, y_train) # ensemble34.fit(x_train, y_train) # ensemble123.fit(x_train, y_train) # ensemble124.fit(x_train, y_train) # ensemble134.fit(x_train, y_train) # ensemble234.fit(x_train, y_train) # ensemble1234.fit(x_train, y_train) # ensemble5678.fit(x_train, y_train) ensemble.fit(x_train, y_train) for clf in (ensemble, ensemble): before_time = time.time() y_pred = clf.predict(x_test) model_time = time.time() - before_time print(clf.__class__.__name__, accuracy_score(y_test, y_pred)) print("Time: ", model_time)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_cifar10_data() y_train2 = tf.keras.utils.to_categorical(y_train, 10) y_test = tf.squeeze(y_test) print("Loading models...") l1_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l1_model, epochs=10, verbose=True) l1_model._estimator_type = "classifier" l2_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l2_model, epochs=10, verbose=True) l2_model._estimator_type = "classifier" l3_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l3_model, epochs=10, verbose=True) l3_model._estimator_type = "classifier" l4_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l4_model, epochs=10, verbose=True) l4_model._estimator_type = "classifier" l5_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l5_model, epochs=10, verbose=True) l5_model._estimator_type = "classifier" l6_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l6_model, epochs=10, verbose=True) l6_model._estimator_type = "classifier" l7_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l7_model, epochs=10, verbose=True) l7_model._estimator_type = "classifier" l8_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l8_model, epochs=10, verbose=True) l8_model._estimator_type = "classifier" l9_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l9_model, epochs=10, verbose=True) l9_model._estimator_type = "classifier" l10_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l10_model, epochs=10, verbose=True) l10_model._estimator_type = "classifier" l1_model.fit(x_train, y_train2) l2_model.fit(x_train, y_train2) l3_model.fit(x_train, y_train2) l4_model.fit(x_train, y_train2) l5_model.fit(x_train, y_train) l6_model.fit(x_train, y_train) l7_model.fit(x_train, y_train) l8_model.fit(x_train, y_train) l9_model.fit(x_train, y_train) l10_model.fit(x_train, y_train) accuracy_scores = [] for clf in (l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model): before_time = time.time() y_pred = clf.predict(x_test) model_time = time.time() - before_time accuracy = accuracy_score(y_test, y_pred) accuracy_scores.append(accuracy) print(clf.__class__.__name__, accuracy) print("Time: ", model_time) # indices = [0, 1] # ensemble12 = VotingClassifier(estimators=[('l1', l1_model), # ('l2', l2_model)], # voting='soft', weights=[accuracy_scores[i] for i in indices]) # indices = [0, 2] # ensemble13 = VotingClassifier(estimators=[('l1', l1_model), # ('l3', l3_model)], # voting='soft', weights=[accuracy_scores[i] for i in indices]) # indices = [0, 3] # ensemble14 = VotingClassifier(estimators=[('l1', l1_model), # ('l4', l4_model)], # voting='soft', weights=[accuracy_scores[i] for i in indices]) # indices = [1, 2] # ensemble23 = VotingClassifier(estimators=[('l2', l2_model), # ('l3', l3_model)], # voting='soft', weights=[accuracy_scores[i] for i in indices]) # indices = [1, 3] # ensemble24 = VotingClassifier(estimators=[('l2', l2_model), # ('l4', l4_model)], # voting='soft', weights=[accuracy_scores[i] for i in indices]) # indices = [2, 3] # ensemble34 = VotingClassifier(estimators=[('l3', l3_model), # ('l4', l4_model)], # voting='soft', weights=[accuracy_scores[i] for i in indices]) # indices = [0, 1, 2] # ensemble123 = VotingClassifier(estimators=[('l1', l1_model), # ('l2', l2_model), # ('l3', l3_model)], # voting='soft', weights=[accuracy_scores[i] for i in indices]) # indices = [0, 1, 3] # ensemble124 = VotingClassifier(estimators=[('l1', l1_model), # ('l2', l2_model), # ('l4', l4_model)], # voting='soft', weights=[accuracy_scores[i] for i in indices]) # indices = [0, 2, 3] # ensemble134 = VotingClassifier(estimators=[('l1', l1_model), # ('l3', l3_model), # ('l4', l4_model)], # voting='soft', weights=[accuracy_scores[i] for i in indices]) # indices = [1, 2, 3] # ensemble234 = VotingClassifier(estimators=[('l2', l2_model), # ('l3', l3_model), # ('l4', l4_model)], # voting='soft', weights=[accuracy_scores[i] for i in indices]) ensemble = VotingClassifier(estimators=[('l1', l1_model), ('l2', l2_model), ('l3', l3_model), ('l4', l4_model), ('l5', l5_model), ('l6', l6_model), ('l7', l7_model), ('l8', l8_model), ('l9', l9_model), ('l10', l10_model)], voting='soft', weights=accuracy_scores, verbose=True) # ensemble12.fit(x_train, y_train) # ensemble13.fit(x_train, y_train) # ensemble14.fit(x_train, y_train) # ensemble23.fit(x_train, y_train) # ensemble24.fit(x_train, y_train) # ensemble34.fit(x_train, y_train) # ensemble123.fit(x_train, y_train) # ensemble124.fit(x_train, y_train) # ensemble134.fit(x_train, y_train) # ensemble234.fit(x_train, y_train) ensemble.fit(x_train, y_train) for clf in (ensemble, ensemble): before_time = time.time() y_pred = clf.predict(x_test) model_time = time.time() - before_time print(clf.__class__.__name__, accuracy_score(y_test, y_pred)) print("Time: ", model_time)