def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_mnist_data() # train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/l1_model') l2_model = tf.keras.models.load_model('models/l2_model') l3_model = tf.keras.models.load_model('models/l3_model') # l4_model = tf.keras.models.load_model('models/l4_model') all_model_accuracies = [] all_model_times = [] all_model_simple_percents = [] all_model_complex_percents = [] for i in range(5): print("Run l1 l2 l3... #" + str(i)) all_conf_values, accuracies, times, simple_percents, complex_percents = run_combinations( l1_model, l2_model, l3_model, x_test, y_test) all_model_accuracies.append(accuracies) all_model_times.append(times) all_model_simple_percents.append(simple_percents) all_model_complex_percents.append(complex_percents) print("All Conf Values:", all_conf_values) print("L1 L2 L3 Accuracies:", np.mean(all_model_accuracies, axis=0)) print("L1 L2 L3 Times:", np.mean(all_model_times, axis=0)) print("L1 L2 L3 Simple Percents:", np.mean(all_model_simple_percents, axis=0)) print("L1 L2 L3 Complex Percents:", np.mean(all_model_complex_percents, axis=0))
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_mnist_data() num_classes = 10 #y_train = tf.keras.utils.to_categorical(y_train, num_classes) train_and_save_models(x_train, y_train)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_mnist_data() #train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/l1_model') l2_model = tf.keras.models.load_model('models/l2_model') l3_model = tf.keras.models.load_model('models/l3_model') # l4_model = tf.keras.models.load_model('models/l4_model') p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations( l1_model, l2_model, x_test, y_test) print("L1 L2 values:", p_yy, p_ny, p_yn, p_nn) print(yn_values) print(ny_values) p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations( l1_model, l3_model, x_test, y_test) print("L1 L3 values:", p_yy, p_ny, p_yn, p_nn) print(yn_values) print(ny_values) p_yy, p_yn, p_ny, p_nn, yn_values, ny_values = run_combinations( l2_model, l3_model, x_test, y_test) print("L2 L3 values:", p_yy, p_ny, p_yn, p_nn) print(yn_values) print(ny_values)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_mnist_data() # train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/l1_model') l2_model = tf.keras.models.load_model('models/l2_model') l3_model = tf.keras.models.load_model('models/l3_model') # l4_model = tf.keras.models.load_model('models/l4_model') # before_time = time.time() # accuracy = l0_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l1_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l2_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l3_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) # before_time = time.time() # accuracy = l4_model.evaluate(x_test, y_test) # print("Time", time.time() - before_time) all_model_accuracies = [] all_model_times = [] all_model_simple_percents = [] all_model_complex_percents = [] all_model_thresholds = [] for i in range(1): print("Run l1 l2... #" + str(i)) all_conf_values, accuracies, times, simple_percents, thresholds = run_combinations( l2_model, l3_model, x_test, y_test) all_model_accuracies.append(accuracies) all_model_times.append(times) all_model_simple_percents.append(simple_percents) all_model_thresholds.append(thresholds) print("All Conf Values:", all_conf_values) print("L1 L2 Accuracies:", np.mean(all_model_accuracies, axis=0)) print("L1 L2 Times:", np.mean(all_model_times, axis=0)) print("L1 L2 Simple Percents:", np.mean(all_model_simple_percents, axis=0)) print("L1 L2 Thresholds:", np.mean(all_model_thresholds, axis=0))
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_mnist_data() y_test = tf.squeeze(y_test) #train_and_save_models(x_train, y_train) print("Loading models...") l1_model = tf.keras.models.load_model('models/mnist/l1_model') l2_model = tf.keras.models.load_model('models/mnist/l2_model') l3_model = tf.keras.models.load_model('models/mnist/l3_model') l4_model = tf.keras.models.load_model('models/mnist/l4_model') l5_model = tf.keras.models.load_model('models/mnist/l5_model') l6_model = tf.keras.models.load_model('models/mnist/l6_model') l7_model = tf.keras.models.load_model('models/mnist/l7_model') l8_model = tf.keras.models.load_model('models/mnist/l8_model') l9_model = tf.keras.models.load_model('models/mnist/l9_model') l10_model = tf.keras.models.load_model('models/mnist/l10_model') # Get dictionary of counts of each class in y_test y_test_np = y_test.numpy() #unique, counts = np.unique(y_test.numpy(), return_counts=True) count_dicts = [] # Set up accuracy grid accuracies = np.zeros((10, 10)) # Iterate over all models and get their predicted outputs models = [l1_model, l2_model, l3_model, l4_model, l5_model, l6_model, l7_model, l8_model, l9_model, l10_model] for i in range(10): model = models[i] model_probs = model.predict(x_test) model_preds = np.argmax(model_probs, axis=1) unique, counts = np.unique(model_preds, return_counts=True) count_dicts.append(dict(zip(unique, counts))) # Iterate over all 10 classes for j in range(10): # Compute the number of times where the prediction matches the test output for that class class_count = len(np.where((model_preds == j) & (y_test_np == j))[0]) accuracies[i][j] = class_count / count_dicts[i][j] print(accuracies)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_mnist_data() #train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/l1_model') l2_model = tf.keras.models.load_model('models/l2_model') l3_model = tf.keras.models.load_model('models/l3_model') # l4_model = tf.keras.models.load_model('models/l4_model') p_yy, p_yn, p_ny, p_nn, yn_sub, ny_sub, nn_sub = run_combinations( l1_model, l2_model, x_test, y_test) print("L1 L2 values:", p_yy, p_ny, p_yn, p_nn) #print("YY:", yy_values) print("YN:", yn_sub / p_yn) print("NY:", ny_sub / p_ny) print("NN:", nn_sub / p_nn) #print("L1: ", l1_values[:1000]) p_yy, p_yn, p_ny, p_nn, yn_sub, ny_sub, nn_sub = run_combinations( l1_model, l3_model, x_test, y_test) print("L1 L3 values:", p_yy, p_ny, p_yn, p_nn) #print("YY:", yy_values) print("YN:", yn_sub / p_yn) print("NY:", ny_sub / p_ny) print("NN:", nn_sub / p_nn) #print("L1: ", l1_values[:1000]) p_yy, p_yn, p_ny, p_nn, yn_sub, ny_sub, nn_sub = run_combinations( l2_model, l3_model, x_test, y_test) print("L2 L3 values:", p_yy, p_ny, p_yn, p_nn) #print("YY:", yy_values) print("YN:", yn_sub / p_yn) print("NY:", ny_sub / p_ny) print("NN:", nn_sub / p_nn)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_mnist_data() y_train2 = tf.keras.utils.to_categorical(y_train, 10) #y_test = tf.squeeze(y_test) print("Loading models...") l1_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l1_model, epochs=5, verbose=True) l1_model._estimator_type = "classifier" l2_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l2_model, epochs=5, verbose=True) l2_model._estimator_type = "classifier" l3_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l3_model, epochs=5, verbose=True) l3_model._estimator_type = "classifier" l4_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l4_model, epochs=5, verbose=True) l4_model._estimator_type = "classifier" ensemble1 = StackingClassifier(estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)], final_estimator=RandomForestClassifier( random_state=42, n_estimators=10)) ensemble2 = StackingClassifier( estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)], final_estimator=RandomForestClassifier(random_state=42)) ensemble3 = StackingClassifier(estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)], final_estimator=RandomForestClassifier( random_state=42, n_estimators=500)) ensemble4 = StackingClassifier( estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)], final_estimator=HistGradientBoostingClassifier(random_state=42, max_iter=10)) ensemble5 = StackingClassifier( estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)], final_estimator=HistGradientBoostingClassifier(random_state=42)) ensemble1.fit(x_train, y_train) #print("L123 Done") ensemble2.fit(x_train, y_train) #print("L124 Done") ensemble3.fit(x_train, y_train) #print("L134 Done") ensemble4.fit(x_train, y_train) #print("L234 Done") ensemble5.fit(x_train, y_train) #print("L234 Done") for clf in (ensemble1, ensemble2, ensemble3, ensemble4, ensemble5): before_time = time.time() y_pred = clf.predict(x_test) model_time = time.time() - before_time print(clf.__class__.__name__, accuracy_score(y_test, y_pred)) print("Time: ", model_time)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_mnist_data() #y_test = tf.squeeze(y_test) print("Loading models...") l1_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l1_model, epochs=5) l1_model._estimator_type = "classifier" l2_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l2_model, epochs=5) l2_model._estimator_type = "classifier" l3_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l3_model, epochs=5) l3_model._estimator_type = "classifier" l4_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l4_model, epochs=5) l4_model._estimator_type = "classifier" ensemble12 = VotingClassifier(estimators=[('l1', l1_model), ('l2', l2_model)], voting='soft') ensemble13 = VotingClassifier(estimators=[('l1', l1_model), ('l3', l3_model)], voting='soft') ensemble14 = VotingClassifier(estimators=[('l1', l1_model), ('l4', l4_model)], voting='soft') ensemble23 = VotingClassifier(estimators=[('l2', l2_model), ('l3', l3_model)], voting='soft') ensemble24 = VotingClassifier(estimators=[('l2', l2_model), ('l4', l4_model)], voting='soft') ensemble34 = VotingClassifier(estimators=[('l3', l3_model), ('l4', l4_model)], voting='soft') ensemble123 = VotingClassifier(estimators=[('l1', l1_model), ('l2', l2_model), ('l3', l3_model)], voting='soft') ensemble124 = VotingClassifier(estimators=[('l1', l1_model), ('l2', l2_model), ('l4', l4_model)], voting='soft') ensemble134 = VotingClassifier(estimators=[('l1', l1_model), ('l3', l3_model), ('l4', l4_model)], voting='soft') ensemble234 = VotingClassifier(estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)], voting='soft') ensemble1234 = VotingClassifier(estimators=[('l1', l1_model), ('l2', l2_model), ('l3', l3_model), ('l4', l4_model)], voting='soft') l1_model.fit(x_train, y_train) l2_model.fit(x_train, y_train) l3_model.fit(x_train, y_train) l4_model.fit(x_train, y_train) ensemble12.fit(x_train, y_train) ensemble13.fit(x_train, y_train) ensemble14.fit(x_train, y_train) ensemble23.fit(x_train, y_train) ensemble24.fit(x_train, y_train) ensemble34.fit(x_train, y_train) ensemble123.fit(x_train, y_train) ensemble124.fit(x_train, y_train) ensemble134.fit(x_train, y_train) ensemble234.fit(x_train, y_train) ensemble1234.fit(x_train, y_train) for clf in (l1_model, l2_model, l3_model, l4_model, ensemble12, ensemble13, ensemble14, ensemble23, ensemble24, ensemble34, ensemble123, ensemble124, ensemble134, ensemble234, ensemble1234): before_time = time.time() y_pred = clf.predict(x_test) model_time = time.time() - before_time print(clf.__class__.__name__, accuracy_score(y_test, y_pred)) print("Time: ", model_time)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_mnist_data() y_train2 = tf.keras.utils.to_categorical(y_train, 10) #y_test = tf.squeeze(y_test) print("Loading models...") l1_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l1_model, epochs=5) l1_model._estimator_type = "classifier" l2_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l2_model, epochs=5) l2_model._estimator_type = "classifier" l3_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l3_model, epochs=5) l3_model._estimator_type = "classifier" l4_model = tf.keras.wrappers.scikit_learn.KerasClassifier( build_fn=models.get_l4_model, epochs=5) l4_model._estimator_type = "classifier" l1_model.fit(x_train, y_train2) l2_model.fit(x_train, y_train2) l3_model.fit(x_train, y_train2) l4_model.fit(x_train, y_train2) accuracy_scores = [] for clf in (l1_model, l2_model, l3_model, l4_model): before_time = time.time() y_pred = clf.predict(x_test) model_time = time.time() - before_time accuracy = accuracy_score(y_test, y_pred) accuracy_scores.append(accuracy) print(clf.__class__.__name__, accuracy) print("Time: ", model_time) indices = [0, 1] ensemble12 = VotingClassifier( estimators=[('l1', l1_model), ('l2', l2_model)], voting='soft', weights=[accuracy_scores[i] for i in indices]) indices = [0, 2] ensemble13 = VotingClassifier( estimators=[('l1', l1_model), ('l3', l3_model)], voting='soft', weights=[accuracy_scores[i] for i in indices]) indices = [0, 3] ensemble14 = VotingClassifier( estimators=[('l1', l1_model), ('l4', l4_model)], voting='soft', weights=[accuracy_scores[i] for i in indices]) indices = [1, 2] ensemble23 = VotingClassifier( estimators=[('l2', l2_model), ('l3', l3_model)], voting='soft', weights=[accuracy_scores[i] for i in indices]) indices = [1, 3] ensemble24 = VotingClassifier( estimators=[('l2', l2_model), ('l4', l4_model)], voting='soft', weights=[accuracy_scores[i] for i in indices]) indices = [2, 3] ensemble34 = VotingClassifier( estimators=[('l3', l3_model), ('l4', l4_model)], voting='soft', weights=[accuracy_scores[i] for i in indices]) indices = [0, 1, 2] ensemble123 = VotingClassifier( estimators=[('l1', l1_model), ('l2', l2_model), ('l3', l3_model)], voting='soft', weights=[accuracy_scores[i] for i in indices]) indices = [0, 1, 3] ensemble124 = VotingClassifier( estimators=[('l1', l1_model), ('l2', l2_model), ('l4', l4_model)], voting='soft', weights=[accuracy_scores[i] for i in indices]) indices = [0, 2, 3] ensemble134 = VotingClassifier( estimators=[('l1', l1_model), ('l3', l3_model), ('l4', l4_model)], voting='soft', weights=[accuracy_scores[i] for i in indices]) indices = [1, 2, 3] ensemble234 = VotingClassifier( estimators=[('l2', l2_model), ('l3', l3_model), ('l4', l4_model)], voting='soft', weights=[accuracy_scores[i] for i in indices]) ensemble1234 = VotingClassifier(estimators=[('l1', l1_model), ('l2', l2_model), ('l3', l3_model), ('l4', l4_model)], voting='soft', weights=accuracy_scores) ensemble12.fit(x_train, y_train) ensemble13.fit(x_train, y_train) ensemble14.fit(x_train, y_train) ensemble23.fit(x_train, y_train) ensemble24.fit(x_train, y_train) ensemble34.fit(x_train, y_train) ensemble123.fit(x_train, y_train) ensemble124.fit(x_train, y_train) ensemble134.fit(x_train, y_train) ensemble234.fit(x_train, y_train) ensemble1234.fit(x_train, y_train) for clf in (l1_model, l2_model, l3_model, l4_model, ensemble12, ensemble13, ensemble14, ensemble23, ensemble24, ensemble34, ensemble123, ensemble124, ensemble134, ensemble234, ensemble1234): before_time = time.time() y_pred = clf.predict(x_test) model_time = time.time() - before_time print(clf.__class__.__name__, accuracy_score(y_test, y_pred)) print("Time: ", model_time)
def main(): print('Loading data...') x_train, y_train, x_test, y_test = helpers.get_mnist_data() #train_and_save_models(x_train, y_train) print("Loading models...") # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/l1_model') l2_model = tf.keras.models.load_model('models/l2_model') l3_model = tf.keras.models.load_model('models/l3_model') # l4_model = tf.keras.models.load_model('models/l4_model') l1_time = 0 l2_time = 0 l3_time = 0 for i in range(5): before_time = time.time() accuracy = l1_model.predict(x_test) l1_time += time.time() - before_time print(l1_time) before_time = time.time() accuracy = l2_model.predict(x_test) l2_time += time.time() - before_time before_time = time.time() accuracy = l3_model.predict(x_test) l3_time += time.time() - before_time print("L1 Time:", l1_time / 5) print("L2 Time:", l2_time / 5) print("L3 Time:", l3_time / 5) l1_l2_accuracies = [] l1_l3_accuracies = [] l2_l3_accuracies = [] l1_l2_times = [] l1_l3_times = [] l2_l3_times = [] l1_l2_percents = [] l1_l3_percents = [] l2_l3_percents = [] for i in range(5): print("Run l1 l2... #" + str(i)) accuracies, times, simplePercents = run_combinations(l1_model, l2_model, x_test, y_test) l1_l2_accuracies.append(accuracies) l1_l2_times.append(times) l1_l2_percents.append(simplePercents) print("Run l1 l3... #" + str(i)) accuracies, times, simplePercents = run_combinations(l1_model, l3_model, x_test, y_test) l1_l3_accuracies.append(accuracies) l1_l3_times.append(times) l1_l3_percents.append(simplePercents) print("Run l2 l3... #" + str(i)) accuracies, times, simplePercents = run_combinations(l2_model, l3_model, x_test, y_test) l2_l3_accuracies.append(accuracies) l2_l3_times.append(times) l2_l3_percents.append(simplePercents) print("L1 L2 Accuracies:", np.mean(l1_l2_accuracies, axis=0)) print("L1 L2 Times:", np.mean(l1_l2_times, axis=0)) print("L1 L2 Simple Percents", np.mean(l1_l2_percents, axis=0)) print("L1 L3 Accuracies:", np.mean(l1_l3_accuracies, axis=0)) print("L1 L3 Times:", np.mean(l1_l3_times, axis=0)) print("L1 L3 Simple Percents", np.mean(l1_l3_percents, axis=0)) print("L2 L3 Accuracies:", np.mean(l2_l3_accuracies, axis=0)) print("L2 L3 Times:", np.mean(l2_l3_times, axis=0)) print("L2 L3 Simple Percents", np.mean(l2_l3_percents, axis=0))
def main(): run_greedy_walk = True run_naive = False # ====================================================================== # Preparing data and trained models print('Loading data...') # Get MNIST data x_train, y_train, x_test, y_test = helpers.get_mnist_data() # For testing model to sort by complexity x_test_1 = x_test[:3333] y_test_1 = y_test[:3333] # For finding optimal confidence value x_test_2 = x_test[3334:6666] y_test_2 = y_test[3334:6666] # For validating optimal confidence value x_test_3 = x_test[6667:10000] y_test_3 = y_test[6667:10000] print('Loading models...') # l0_model = tf.keras.models.load_model('models/l0_model') l1_model = tf.keras.models.load_model('models/l1_model') l2_model = tf.keras.models.load_model('models/l2_model') l3_model = tf.keras.models.load_model('models/l3_model') print('Storing models in Graph object...') # Load saved models and store as Graph.Model object (with corresponding complexity) models_list = [] for i in range(1, 4): current_model = tf.keras.models.load_model('models/l' + str(i) + '_model') current_model_object = Graph.Model(current_model) before_time = time.time() results = current_model.evaluate(x_test_1, y_test_1, verbose=0) after_time = time.time() current_model_object.accuracy = results[1] current_model_object.time = after_time - before_time current_model_object.complexity_index = i - 1 print('complexity_index:', current_model_object.complexity_index, \ 'accuracy:', current_model_object.accuracy, 'time: ', current_model_object.time) models_list.append(current_model_object) # Create graph object and add models sorted based on complexity graph = Graph.Graph(models_list) # ====================================================================== # Running Greedy Walk: if run_greedy_walk: print('Running Greedy Walk...') greedy_output = [] # for current_model_search_time in np.flip(np.arange(0.2, 0.5, 0.05)): print("=========================================================") current_model_search_time = 0.45 print("Greedy:", current_model_search_time) output, search_time, best_node_accuracy = greedy_walk(graph, 20, \ current_model_search_time, x_test_2, y_test_2, x_test_3, y_test_3) if output == None: print('Greedy walk did not find any viable pairing') greedy_output.append(None) else: print('Greedy walk found', output.complexities, 'in', search_time, 'seconds') greedy_output.append((output.complexities, search_time, best_node_accuracy, output.optimal_confidence_value)) print(greedy_output) # ====================================================================== # Running Naive: if run_naive: naive_output = [] # for current_model_search_time in np.arange(0.2, 0.5, 0.05): current_model_search_time = 0.35 print("=========================================================") print("Naive:", current_model_search_time) output, search_time, best_node_accuracy = naive_search(models_list, current_model_search_time,\ x_test_2, y_test_2, x_test_3, y_test_3) if output == None: print('Naive did not any find viable pairing') naive_output.append(None) else: print('Naive found', output.complexities, 'in', search_time, 'seconds') naive_output.append((output.complexities, search_time, best_node_accuracy, output.optimal_confidence_value)) print(naive_output)