def run_models_with_noise(models, noise):
goal_list = [
"g_1_make_coffee", "g_1_make_tea", "g_2_add_grounds", "g_2_add_cream",
"g_2_add_sugar", "g_2_drink", "g_2_dip_teabag"
]
all_action_targets_str = utils.flatten_onelevel(
[tce.action_list[goal] for goal in goal_list])
all_action_targets = [
utils.str_to_onehot(action, tce.TeaCoffeeData.actions_list)
for action in all_action_targets_str
]
# Gather actual outputs
all_outputs = []
for model in models:
all_outputs.append(run_model_with_noise(model, noise))
# Now check which actions and which goals they got correctly
num_bad_actions = np.zeros(len(all_action_targets))
for output in all_outputs:
for i, action in enumerate(output):
if (all_action_targets[i] != action).any():
num_bad_actions[i] += 1
print(all_action_targets_str[i],
utils.onehot_to_str(action, tce.TeaCoffeeData.actions_list))
return num_bad_actions
def accuracy_test_with_goals(model, test_number=None):
hidden_activation = []
all_choices = []
for j, sequence in enumerate(seqs):
goal = goals[j]
seq_choices = []
all_choices.append(seq_choices)
inputs = utils.liststr_to_onehot(sequence[:-1], all_inputs)
targets = utils.liststr_to_onehot(sequence[1:], all_outputs)
model.action = np.zeros((1, model.size_action), dtype=np.float32)
# run the network
with tf.GradientTape() as tape:
# Initialize context with random/uniform values.
model.context = np.zeros((1, model.size_hidden), dtype=np.float32)
model.goal1 = goal[0]
# Reset the previous action
for i in range(len(targets)):
model.action = np.zeros((1, model.size_action),
dtype=np.float32)
observation = inputs[i].reshape(1, -1)
model.feedforward(observation)
hidden_activation.append(model.context)
# Get some statistics about what was correct and what wasn't
choice = np.array(model.h_action_wta).reshape(
(-1, len(targets[0])))
model.h_action_wta.clear()
seq_choices.append(choice)
# Now evaluate accuracy:
accuracy = np.zeros((len(seq1) - 1))
accuracy_weighted = np.zeros((len(seq1) - 1))
for i in range(len(all_choices)):
targets = utils.liststr_to_onehot(seqs[i][1:], all_outputs)
for j in range(len(targets)):
if (all_choices[i][0][j] == targets[j]).all():
accuracy_weighted[j] += 1 * sequence_probabilities[i]
accuracy[j] += 1 / len(all_choices)
optimal = np.array_equal(accuracy_weighted, optimal_accuracy_goals)
if test_number is None:
print(accuracy, accuracy_weighted, optimal)
else:
print("{0} ({1}) - network {2} -- {3}".format(accuracy,
accuracy_weighted,
test_number, optimal))
if not optimal:
for i in range(len(seqs)):
print([
utils.onehot_to_str(all_choices[i][0][j], all_outputs)
for j in range(len(targets))
])
return hidden_activation, optimal
def test_one_sequence(model,
sequence_num,
turn_goal_step=None,
goal_to_turn=None):
hidden_activation = []
all_choices = []
results = []
for trials in range(100):
sequence = pnas2018task.seqs[sequence_num]
goal = pnas2018task.goals[sequence_num]
seq_choices = []
all_choices.append(seq_choices)
inputs = utils.liststr_to_onehot(sequence[:-1],
pnas2018task.all_inputs)
targets = utils.liststr_to_onehot(sequence[1:],
pnas2018task.all_outputs)
model.action = np.zeros((1, model.size_action), dtype=np.float32)
# run the network
with tf.GradientTape() as tape:
# Initialize context with random/uniform values.
model.context = np.float32(
np.abs(
np.random.randint(0, 2, (1, model.size_hidden)) -
0.1)) # np.zeros((1, model.size_hidden), dtype=np.float32)
model.goal1 = goal[0]
# Reset the previous action
for i in range(len(targets)):
if i == turn_goal_step:
model.goal1 = goal_to_turn
model.action = np.zeros((1, model.size_action),
dtype=np.float32)
observation = inputs[i].reshape(1, -1)
model.feedforward(observation)
hidden_activation.append(model.context)
# Get some statistics about what was correct and what wasn't
choice = np.array(model.h_action_wta).reshape(
(-1, len(targets[0])))
model.clear_history()
results.append(choice)
# Now, count the number of unique result sequences, and the number of occurences for each unique sequence
unique_results = []
unique_results_counts = []
for result in results:
unique = True
for i, unique_result in enumerate(unique_results):
if np.array_equal(result, unique_result):
unique_results_counts[i] += 1
unique = False
break
if unique:
unique_results.append(result)
unique_results_counts.append(1)
# Sort in order of frequency
unique_results = [
unique_result
for (_, unique_result
) in sorted(zip(unique_results_counts, unique_results),
key=lambda pair: pair[0],
reverse=True)
]
unique_results_counts = sorted(unique_results_counts, reverse=True)
# Print the target sequence
full_sequence_str = ""
for row in targets:
full_sequence_str += utils.onehot_to_str(
row, pnas2018task.all_outputs) + "; "
print("target: " + full_sequence_str)
# Now print the results
for i, unique_result in enumerate(unique_results):
full_sequence_str = ""
for row in unique_result:
full_sequence_str += utils.onehot_to_str(
row, pnas2018task.all_outputs) + "; "
print(str(unique_results_counts[i]) + "%: " + full_sequence_str)
return hidden_activation
def accuracy_test_predictive(model, test_number=None):
hidden_activation = []
all_choices = []
all_predictions = []
for sequence in pnas2018task.seqs:
seq_choices = []
seq_predictions = []
all_predictions.append(seq_predictions)
all_choices.append(seq_choices)
inputs = utils.liststr_to_onehot(sequence[:-1], pnas2018task.all_inputs)
action_targets = utils.liststr_to_onehot(sequence[1:], pnas2018task.all_outputs)
prediction_targets = utils.liststr_to_onehot(sequence[1:], pnas2018task.all_inputs)
model.action = np.zeros((1, model.size_action), dtype=np.float32)
# run the network
with tf.GradientTape() as tape:
model.context = np.zeros((1, model.size_hidden), dtype=np.float32)
model.prediction_linear = np.zeros((1, model.size_observation), dtype=np.float32) #initial prediction = 0,
# Reset the previous action
for i in range(len(action_targets)):
model.action = np.zeros((1, model.size_action), dtype=np.float32)
observation = inputs[i].reshape(1, -1)
model.feedforward(observation)
hidden_activation.append(model.context)
# Get some statistics about what was correct and what wasn't
choice = np.array(model.h_action_wta).reshape((-1, len(action_targets[0])))
prediction = np.array(model.h_prediction_wta).reshape((-1, len(prediction_targets[0])))
model.h_action_wta.clear()
model.h_prediction_wta.clear()
seq_choices.append(choice)
seq_predictions.append(prediction)
# Now evaluate accuracy:
optimal_accuracy = np.asarray([.5, .5, 1., 1., 1., 1.])
accuracy = np.zeros((len(pnas2018task.seq1) - 1))
accuracy_weighted = np.zeros((len(pnas2018task.seq1) - 1))
for i in range(len(all_choices)):
action_targets = utils.liststr_to_onehot(pnas2018task.seqs[i][1:], pnas2018task.all_outputs)
for j in range(len(action_targets)):
if (all_choices[i][0][j] == action_targets[j]).all():
accuracy_weighted[j] += 1 * pnas2018task.sequence_probabilities[i]
accuracy[j] += 1/len(all_choices)
optimal_actions = np.array_equal(accuracy_weighted, optimal_accuracy)
optimal_accuracy_preds = [.5, .5, 1, 1, 1, 1]
accuracy_preds = np.zeros((len(pnas2018task.seq1) - 1))
accuracy_preds_weighted = np.zeros((len(pnas2018task.seq1) - 1))
for i in range(len(all_predictions)):
prediction_targets = utils.liststr_to_onehot(pnas2018task.seqs[i][1:], pnas2018task.all_inputs)
for j in range(len(prediction_targets)):
if (all_predictions[i][0][j] == prediction_targets[j]).all():
accuracy_preds_weighted[j] += 1 * pnas2018task.sequence_probabilities[i]
accuracy_preds[j] += 1/len(all_predictions)
optimal_predictions = np.array_equal(accuracy_preds_weighted, optimal_accuracy_preds)
if test_number is None:
print(accuracy, accuracy_weighted, optimal_actions, accuracy_preds, accuracy_preds_weighted, optimal_predictions)
else:
print("Actions: {0} ({1}) - network {2} -- {3}".format(accuracy, accuracy_weighted, test_number, optimal_actions))
if not optimal_actions or not optimal_predictions:
print("actions:")
for i in range(len(pnas2018task.seqs)):
print([utils.onehot_to_str(all_choices[i][0][j], pnas2018task.all_outputs) for j in range(len(action_targets))])
print("predictions:")
for i in range(len(pnas2018task.seqs)):
print([utils.onehot_to_str(all_predictions[i][0][j], pnas2018task.all_inputs) for j in range(len(prediction_targets))])
return hidden_activation, optimal_actions and optimal_predictions
def accuracy_test_predictive(model, test_number=None, type='sigmoid'):
inputs_str = [
"start", "coffee", "milk", "cream", "water", "stir", "tea", "serve",
"sugar", "end"
]
outputs_str = [
"start", "coffee", "milk", "cream", "water", "stir", "tea", "servetea",
"servecoffee", "sugar", "end"
]
seq1in = ['start', 'coffee', 'water', 'stir', 'cream', 'serve',
'end'] # 60%
seq1t = [
'start', 'coffee', 'water', 'stir', 'cream', 'servecoffee', 'end'
] # 60%
seq2in = ['start', 'coffee', 'water', 'stir', 'milk', 'serve',
'end'] # 20%
seq2t = ['start', 'coffee', 'water', 'stir', 'milk', 'servecoffee',
'end'] # 20%
seq3in = ['start', 'tea', 'water', 'stir', 'sugar', 'serve', 'end'] # 20%
seq3t = ['start', 'tea', 'water', 'stir', 'sugar', 'servetea',
'end'] # 20%
inputs_seqs = [seq1in, seq2in, seq3in]
target_seqs = [seq1t, seq2t, seq3t]
hidden_activation = []
all_choices = []
all_predictions = []
for i in range(len(inputs_seqs)):
sequence_i = inputs_seqs[i]
sequence_t = target_seqs[i]
seq_choices = []
seq_predictions = []
all_predictions.append(seq_predictions)
all_choices.append(seq_choices)
inputs = utils.liststr_to_onehot(sequence_i[:-1], inputs_str)
action_targets = utils.liststr_to_onehot(sequence_t[:-1], outputs_str)
prediction_targets = utils.liststr_to_onehot(sequence_i[1:],
inputs_str)
model.action = np.zeros((1, model.size_action), dtype=np.float32)
# run the network
with tf.GradientTape() as tape:
model.context = np.zeros((1, model.size_hidden), dtype=np.float32)
model.prediction_linear = np.zeros(
(1, model.size_observation),
dtype=np.float32) #initial prediction = 0,
# Reset the previous action
for i in range(len(action_targets)):
model.action = np.zeros((1, model.size_action),
dtype=np.float32)
observation = inputs[i].reshape(1, -1)
model.feedforward(observation, type)
hidden_activation.append(model.context)
# Get some statistics about what was correct and what wasn't
choice = np.array(model.h_action_wta).reshape(
(-1, len(action_targets[0])))
prediction = np.array(model.h_prediction_wta).reshape(
(-1, len(prediction_targets[0])))
model.h_action_wta.clear()
model.h_prediction_wta.clear()
seq_choices.append(choice)
seq_predictions.append(prediction)
# Now evaluate accuracy:
optimal_accuracy = np.asarray([1., 1., 1., 1., 1., 1.])
accuracy = np.zeros((len(seq1) - 1))
accuracy_weighted = np.zeros((len(seq1) - 1))
for i in range(len(all_choices)):
action_targets = utils.liststr_to_onehot(target_seqs[i][:-1],
outputs_str)
for j in range(len(action_targets)):
if (all_choices[i][0][j] == action_targets[j]).all():
accuracy_weighted[j] += 1 * sequence_probabilities[i]
accuracy[j] += 1 / len(all_choices)
optimal_actions = np.array_equal(accuracy_weighted, optimal_accuracy)
optimal_accuracy_preds = [.8, 1, 1, .8, 1, 1]
accuracy_preds = np.zeros((len(seq1) - 1))
accuracy_preds_weighted = np.zeros((len(seq1) - 1))
for i in range(len(all_predictions)):
prediction_targets = utils.liststr_to_onehot(inputs_seqs[i][1:],
inputs_str)
for j in range(len(prediction_targets)):
if (all_predictions[i][0][j] == prediction_targets[j]).all():
accuracy_preds_weighted[j] += 1 * sequence_probabilities[i]
accuracy_preds[j] += 1 / len(all_predictions)
optimal_predictions = np.array_equal(accuracy_preds_weighted,
optimal_accuracy_preds)
if test_number is None:
print(accuracy, accuracy_weighted, optimal_actions, accuracy_preds,
accuracy_preds_weighted, optimal_predictions)
else:
print("Actions: {0} ({1}) - network {2} -- {3}".format(
accuracy, accuracy_weighted, test_number, optimal_actions
and optimal_predictions))
if not optimal_actions or not optimal_predictions:
print("actions:")
for i in range(len(seqs)):
print([
utils.onehot_to_str(all_choices[i][0][j], outputs_str)
for j in range(len(action_targets))
])
print("predictions:")
for i in range(len(seqs)):
print([
utils.onehot_to_str(all_predictions[i][0][j], inputs_str)
for j in range(len(prediction_targets))
])
return hidden_activation, optimal_actions and optimal_predictions
def flatten(grad, inp):
return [x[np.argmax(l)] for x, l in zip(grad, inp)]
easy_neg_grad = conv_net.get_grads(
sess, np.array([list(x[2]) for x in hard_negatives[-1:]]))
easy_pos_grad = conv_net.get_grads(
sess, np.array([list(x[2]) for x in hard_positives[-1:]]))
neg_flat = flatten(easy_neg_grad, hard_negatives[-1][2])
pos_flat = flatten(easy_pos_grad, hard_positives[-1][2])
#print(easy_pos_grad)
print(utils.onehot_to_str(hard_positives[-1][2]))
print(pos_flat)
print(utils.onehot_to_str(hard_negatives[-1][2]))
#print(flatten(easy_pos_grad, hard_positives[-1][2]))
heatmap(neg_flat, utils.onehot_to_str(hard_negatives[-1][2]),
'saliency_negative.png')
heatmap(pos_flat, utils.onehot_to_str(hard_positives[-1][2]),
'saliency_positive.png')
'''#print(all_results[:100])
#print(hard_positives[:100])
auroc = conv_net.calc_roc(sess, 'conv_auroc.png')
print("ROC AUC: %g" % auroc)
auprc = conv_net.calc_auprc(sess, 'conv_auprc.png')
print("PR AUC: %g" % auprc)