def run_models_with_noise(models, noise): goal_list = [ "g_1_make_coffee", "g_1_make_tea", "g_2_add_grounds", "g_2_add_cream", "g_2_add_sugar", "g_2_drink", "g_2_dip_teabag" ] all_action_targets_str = utils.flatten_onelevel( [tce.action_list[goal] for goal in goal_list]) all_action_targets = [ utils.str_to_onehot(action, tce.TeaCoffeeData.actions_list) for action in all_action_targets_str ] # Gather actual outputs all_outputs = [] for model in models: all_outputs.append(run_model_with_noise(model, noise)) # Now check which actions and which goals they got correctly num_bad_actions = np.zeros(len(all_action_targets)) for output in all_outputs: for i, action in enumerate(output): if (all_action_targets[i] != action).any(): num_bad_actions[i] += 1 print(all_action_targets_str[i], utils.onehot_to_str(action, tce.TeaCoffeeData.actions_list)) return num_bad_actions
def accuracy_test_with_goals(model, test_number=None): hidden_activation = [] all_choices = [] for j, sequence in enumerate(seqs): goal = goals[j] seq_choices = [] all_choices.append(seq_choices) inputs = utils.liststr_to_onehot(sequence[:-1], all_inputs) targets = utils.liststr_to_onehot(sequence[1:], all_outputs) model.action = np.zeros((1, model.size_action), dtype=np.float32) # run the network with tf.GradientTape() as tape: # Initialize context with random/uniform values. model.context = np.zeros((1, model.size_hidden), dtype=np.float32) model.goal1 = goal[0] # Reset the previous action for i in range(len(targets)): model.action = np.zeros((1, model.size_action), dtype=np.float32) observation = inputs[i].reshape(1, -1) model.feedforward(observation) hidden_activation.append(model.context) # Get some statistics about what was correct and what wasn't choice = np.array(model.h_action_wta).reshape( (-1, len(targets[0]))) model.h_action_wta.clear() seq_choices.append(choice) # Now evaluate accuracy: accuracy = np.zeros((len(seq1) - 1)) accuracy_weighted = np.zeros((len(seq1) - 1)) for i in range(len(all_choices)): targets = utils.liststr_to_onehot(seqs[i][1:], all_outputs) for j in range(len(targets)): if (all_choices[i][0][j] == targets[j]).all(): accuracy_weighted[j] += 1 * sequence_probabilities[i] accuracy[j] += 1 / len(all_choices) optimal = np.array_equal(accuracy_weighted, optimal_accuracy_goals) if test_number is None: print(accuracy, accuracy_weighted, optimal) else: print("{0} ({1}) - network {2} -- {3}".format(accuracy, accuracy_weighted, test_number, optimal)) if not optimal: for i in range(len(seqs)): print([ utils.onehot_to_str(all_choices[i][0][j], all_outputs) for j in range(len(targets)) ]) return hidden_activation, optimal
def test_one_sequence(model, sequence_num, turn_goal_step=None, goal_to_turn=None): hidden_activation = [] all_choices = [] results = [] for trials in range(100): sequence = pnas2018task.seqs[sequence_num] goal = pnas2018task.goals[sequence_num] seq_choices = [] all_choices.append(seq_choices) inputs = utils.liststr_to_onehot(sequence[:-1], pnas2018task.all_inputs) targets = utils.liststr_to_onehot(sequence[1:], pnas2018task.all_outputs) model.action = np.zeros((1, model.size_action), dtype=np.float32) # run the network with tf.GradientTape() as tape: # Initialize context with random/uniform values. model.context = np.float32( np.abs( np.random.randint(0, 2, (1, model.size_hidden)) - 0.1)) # np.zeros((1, model.size_hidden), dtype=np.float32) model.goal1 = goal[0] # Reset the previous action for i in range(len(targets)): if i == turn_goal_step: model.goal1 = goal_to_turn model.action = np.zeros((1, model.size_action), dtype=np.float32) observation = inputs[i].reshape(1, -1) model.feedforward(observation) hidden_activation.append(model.context) # Get some statistics about what was correct and what wasn't choice = np.array(model.h_action_wta).reshape( (-1, len(targets[0]))) model.clear_history() results.append(choice) # Now, count the number of unique result sequences, and the number of occurences for each unique sequence unique_results = [] unique_results_counts = [] for result in results: unique = True for i, unique_result in enumerate(unique_results): if np.array_equal(result, unique_result): unique_results_counts[i] += 1 unique = False break if unique: unique_results.append(result) unique_results_counts.append(1) # Sort in order of frequency unique_results = [ unique_result for (_, unique_result ) in sorted(zip(unique_results_counts, unique_results), key=lambda pair: pair[0], reverse=True) ] unique_results_counts = sorted(unique_results_counts, reverse=True) # Print the target sequence full_sequence_str = "" for row in targets: full_sequence_str += utils.onehot_to_str( row, pnas2018task.all_outputs) + "; " print("target: " + full_sequence_str) # Now print the results for i, unique_result in enumerate(unique_results): full_sequence_str = "" for row in unique_result: full_sequence_str += utils.onehot_to_str( row, pnas2018task.all_outputs) + "; " print(str(unique_results_counts[i]) + "%: " + full_sequence_str) return hidden_activation
def accuracy_test_predictive(model, test_number=None): hidden_activation = [] all_choices = [] all_predictions = [] for sequence in pnas2018task.seqs: seq_choices = [] seq_predictions = [] all_predictions.append(seq_predictions) all_choices.append(seq_choices) inputs = utils.liststr_to_onehot(sequence[:-1], pnas2018task.all_inputs) action_targets = utils.liststr_to_onehot(sequence[1:], pnas2018task.all_outputs) prediction_targets = utils.liststr_to_onehot(sequence[1:], pnas2018task.all_inputs) model.action = np.zeros((1, model.size_action), dtype=np.float32) # run the network with tf.GradientTape() as tape: model.context = np.zeros((1, model.size_hidden), dtype=np.float32) model.prediction_linear = np.zeros((1, model.size_observation), dtype=np.float32) #initial prediction = 0, # Reset the previous action for i in range(len(action_targets)): model.action = np.zeros((1, model.size_action), dtype=np.float32) observation = inputs[i].reshape(1, -1) model.feedforward(observation) hidden_activation.append(model.context) # Get some statistics about what was correct and what wasn't choice = np.array(model.h_action_wta).reshape((-1, len(action_targets[0]))) prediction = np.array(model.h_prediction_wta).reshape((-1, len(prediction_targets[0]))) model.h_action_wta.clear() model.h_prediction_wta.clear() seq_choices.append(choice) seq_predictions.append(prediction) # Now evaluate accuracy: optimal_accuracy = np.asarray([.5, .5, 1., 1., 1., 1.]) accuracy = np.zeros((len(pnas2018task.seq1) - 1)) accuracy_weighted = np.zeros((len(pnas2018task.seq1) - 1)) for i in range(len(all_choices)): action_targets = utils.liststr_to_onehot(pnas2018task.seqs[i][1:], pnas2018task.all_outputs) for j in range(len(action_targets)): if (all_choices[i][0][j] == action_targets[j]).all(): accuracy_weighted[j] += 1 * pnas2018task.sequence_probabilities[i] accuracy[j] += 1/len(all_choices) optimal_actions = np.array_equal(accuracy_weighted, optimal_accuracy) optimal_accuracy_preds = [.5, .5, 1, 1, 1, 1] accuracy_preds = np.zeros((len(pnas2018task.seq1) - 1)) accuracy_preds_weighted = np.zeros((len(pnas2018task.seq1) - 1)) for i in range(len(all_predictions)): prediction_targets = utils.liststr_to_onehot(pnas2018task.seqs[i][1:], pnas2018task.all_inputs) for j in range(len(prediction_targets)): if (all_predictions[i][0][j] == prediction_targets[j]).all(): accuracy_preds_weighted[j] += 1 * pnas2018task.sequence_probabilities[i] accuracy_preds[j] += 1/len(all_predictions) optimal_predictions = np.array_equal(accuracy_preds_weighted, optimal_accuracy_preds) if test_number is None: print(accuracy, accuracy_weighted, optimal_actions, accuracy_preds, accuracy_preds_weighted, optimal_predictions) else: print("Actions: {0} ({1}) - network {2} -- {3}".format(accuracy, accuracy_weighted, test_number, optimal_actions)) if not optimal_actions or not optimal_predictions: print("actions:") for i in range(len(pnas2018task.seqs)): print([utils.onehot_to_str(all_choices[i][0][j], pnas2018task.all_outputs) for j in range(len(action_targets))]) print("predictions:") for i in range(len(pnas2018task.seqs)): print([utils.onehot_to_str(all_predictions[i][0][j], pnas2018task.all_inputs) for j in range(len(prediction_targets))]) return hidden_activation, optimal_actions and optimal_predictions
def accuracy_test_predictive(model, test_number=None, type='sigmoid'): inputs_str = [ "start", "coffee", "milk", "cream", "water", "stir", "tea", "serve", "sugar", "end" ] outputs_str = [ "start", "coffee", "milk", "cream", "water", "stir", "tea", "servetea", "servecoffee", "sugar", "end" ] seq1in = ['start', 'coffee', 'water', 'stir', 'cream', 'serve', 'end'] # 60% seq1t = [ 'start', 'coffee', 'water', 'stir', 'cream', 'servecoffee', 'end' ] # 60% seq2in = ['start', 'coffee', 'water', 'stir', 'milk', 'serve', 'end'] # 20% seq2t = ['start', 'coffee', 'water', 'stir', 'milk', 'servecoffee', 'end'] # 20% seq3in = ['start', 'tea', 'water', 'stir', 'sugar', 'serve', 'end'] # 20% seq3t = ['start', 'tea', 'water', 'stir', 'sugar', 'servetea', 'end'] # 20% inputs_seqs = [seq1in, seq2in, seq3in] target_seqs = [seq1t, seq2t, seq3t] hidden_activation = [] all_choices = [] all_predictions = [] for i in range(len(inputs_seqs)): sequence_i = inputs_seqs[i] sequence_t = target_seqs[i] seq_choices = [] seq_predictions = [] all_predictions.append(seq_predictions) all_choices.append(seq_choices) inputs = utils.liststr_to_onehot(sequence_i[:-1], inputs_str) action_targets = utils.liststr_to_onehot(sequence_t[:-1], outputs_str) prediction_targets = utils.liststr_to_onehot(sequence_i[1:], inputs_str) model.action = np.zeros((1, model.size_action), dtype=np.float32) # run the network with tf.GradientTape() as tape: model.context = np.zeros((1, model.size_hidden), dtype=np.float32) model.prediction_linear = np.zeros( (1, model.size_observation), dtype=np.float32) #initial prediction = 0, # Reset the previous action for i in range(len(action_targets)): model.action = np.zeros((1, model.size_action), dtype=np.float32) observation = inputs[i].reshape(1, -1) model.feedforward(observation, type) hidden_activation.append(model.context) # Get some statistics about what was correct and what wasn't choice = np.array(model.h_action_wta).reshape( (-1, len(action_targets[0]))) prediction = np.array(model.h_prediction_wta).reshape( (-1, len(prediction_targets[0]))) model.h_action_wta.clear() model.h_prediction_wta.clear() seq_choices.append(choice) seq_predictions.append(prediction) # Now evaluate accuracy: optimal_accuracy = np.asarray([1., 1., 1., 1., 1., 1.]) accuracy = np.zeros((len(seq1) - 1)) accuracy_weighted = np.zeros((len(seq1) - 1)) for i in range(len(all_choices)): action_targets = utils.liststr_to_onehot(target_seqs[i][:-1], outputs_str) for j in range(len(action_targets)): if (all_choices[i][0][j] == action_targets[j]).all(): accuracy_weighted[j] += 1 * sequence_probabilities[i] accuracy[j] += 1 / len(all_choices) optimal_actions = np.array_equal(accuracy_weighted, optimal_accuracy) optimal_accuracy_preds = [.8, 1, 1, .8, 1, 1] accuracy_preds = np.zeros((len(seq1) - 1)) accuracy_preds_weighted = np.zeros((len(seq1) - 1)) for i in range(len(all_predictions)): prediction_targets = utils.liststr_to_onehot(inputs_seqs[i][1:], inputs_str) for j in range(len(prediction_targets)): if (all_predictions[i][0][j] == prediction_targets[j]).all(): accuracy_preds_weighted[j] += 1 * sequence_probabilities[i] accuracy_preds[j] += 1 / len(all_predictions) optimal_predictions = np.array_equal(accuracy_preds_weighted, optimal_accuracy_preds) if test_number is None: print(accuracy, accuracy_weighted, optimal_actions, accuracy_preds, accuracy_preds_weighted, optimal_predictions) else: print("Actions: {0} ({1}) - network {2} -- {3}".format( accuracy, accuracy_weighted, test_number, optimal_actions and optimal_predictions)) if not optimal_actions or not optimal_predictions: print("actions:") for i in range(len(seqs)): print([ utils.onehot_to_str(all_choices[i][0][j], outputs_str) for j in range(len(action_targets)) ]) print("predictions:") for i in range(len(seqs)): print([ utils.onehot_to_str(all_predictions[i][0][j], inputs_str) for j in range(len(prediction_targets)) ]) return hidden_activation, optimal_actions and optimal_predictions
def flatten(grad, inp): return [x[np.argmax(l)] for x, l in zip(grad, inp)] easy_neg_grad = conv_net.get_grads( sess, np.array([list(x[2]) for x in hard_negatives[-1:]])) easy_pos_grad = conv_net.get_grads( sess, np.array([list(x[2]) for x in hard_positives[-1:]])) neg_flat = flatten(easy_neg_grad, hard_negatives[-1][2]) pos_flat = flatten(easy_pos_grad, hard_positives[-1][2]) #print(easy_pos_grad) print(utils.onehot_to_str(hard_positives[-1][2])) print(pos_flat) print(utils.onehot_to_str(hard_negatives[-1][2])) #print(flatten(easy_pos_grad, hard_positives[-1][2])) heatmap(neg_flat, utils.onehot_to_str(hard_negatives[-1][2]), 'saliency_negative.png') heatmap(pos_flat, utils.onehot_to_str(hard_positives[-1][2]), 'saliency_positive.png') '''#print(all_results[:100]) #print(hard_positives[:100]) auroc = conv_net.calc_roc(sess, 'conv_auroc.png') print("ROC AUC: %g" % auroc) auprc = conv_net.calc_auprc(sess, 'conv_auprc.png') print("PR AUC: %g" % auprc)