def test_prepare_extended_data_inference():
card_sequences, player_hands, hands_to_predict = ld.prepare_extended_data_inference(
game_data_dic, num_samples=26)
assert len(player_hands) == 26
opp_hands = None
chosen_seq = None
pl_hand = None
for seq, hand, opp in zip(card_sequences, player_hands,
range(len(hands_to_predict))):
if np.array_equal(seq[21][:32], np.zeros(32)) and not np.array_equal(
seq[20][:32], np.zeros(32)):
chosen_seq = seq
pl_hand = hand
opp_hands = hands_to_predict[opp]
assert opp_hands is not None
assert chosen_seq is not None
assert pl_hand is not None
opp_hand_1 = enc.decode_hand_inference(opp_hands[:32])
opp_hand_2 = enc.decode_hand_inference(opp_hands[32:64])
opp_hand_3 = enc.decode_hand_inference(opp_hands[64:])
assert set(opp_hand_1) == {(TEN, BELLS), (KING, HEARTS), (OBER, BELLS)}
assert set(opp_hand_2) == {(NINE, ACORNS), (OBER, HEARTS), (EIGHT, HEARTS)}
assert set(opp_hand_3) == {(NINE, BELLS), (EIGHT, BELLS), (SEVEN, BELLS)}
assert set(enc.decode_on_hot_hand(pl_hand)) == {(3, 2), (3, 1), (0, 1),
(7, 3), (6, 3), (2, 2),
(0, 2), (7, 0)}
assert chosen_seq[20][31] == 1
assert chosen_seq[20][33] == 1
def evaluate_model_on_testdata(model,
filepath,
extended_model=True,
threshold=0.7):
num_games = num_games_in_file(filepath)
with open(filepath, 'rb') as f:
count_correct_positives = 0
count_false_positives = 0
num_pred = 0
for num in range(num_games):
data_dic = pickle.load(f)
if extended_model:
card_sequences, aux_input_hands, hands_to_predict = prepare_extended_data_inference(
data_dic, num_samples=26)
y_list = hands_to_predict
for i in range(26):
x = [
np.array([card_sequences[i]]),
np.array([aux_input_hands[i]])
]
y = y_list[i]
predictions = model.predict(x)[0]
top_indices = top_k_indices(predictions, k=96)
correct_indices = np.where(y == 1)[0]
for index in top_indices:
if predictions[top_indices[index]] > threshold:
num_pred += 1
if top_indices[index] in correct_indices:
count_correct_positives += 1
else:
count_false_positives += 1
return count_correct_positives / (count_correct_positives +
count_false_positives), num_pred
def evaluate_model_on_testdata(model,
filepath,
extended_model=True,
threshold=0.7):
num_games = num_games_in_file(filepath)
with open(filepath, 'rb') as f:
count_best_card = 0
count_two_best = 0
count_three_best = 0
count_four_best = 0
count_five_best = 0
count_best_all_hands = 0
count_two_all_hands = 0
count_three_all_hands = 0
count_four_all_hands = 0
count_five_all_hands = 0
count_num_correct_in_top_5 = 0
count_num_correct_in_top_5_all_hands = 0
count_correct_positives = 0
count_correct_positives_all_hands = 0
count_false_positives = 0
count_false_positives_all_hands = 0
for num in range(num_games):
data_dic = pickle.load(f)
if extended_model:
card_sequences, aux_input_hands, hands_to_predict = prepare_extended_data_inference(
data_dic, num_samples=26)
y_list = hands_to_predict
for i in range(26):
x = [
np.array([card_sequences[i]]),
np.array([aux_input_hands[i]])
]
y = y_list[i]
predictions = model.predict(x)[0]
top_indices = top_k_indices(predictions, k=5)
correct_indices = np.where(y == 1)[0]
num_correct = 0
if top_indices[0] in correct_indices:
count_best_card += 1
num_correct += 1
if predictions[top_indices[0]] > threshold:
count_correct_positives += 1
elif predictions[top_indices[0]] > threshold:
count_false_positives += 1
if top_indices[1] in correct_indices:
count_two_best += 1
num_correct += 1
if predictions[top_indices[1]] > threshold:
count_correct_positives += 1
elif predictions[top_indices[1]] > threshold:
count_false_positives += 1
if top_indices[2] in correct_indices:
count_three_best += 1
num_correct += 1
if predictions[top_indices[2]] > threshold:
count_correct_positives += 1
elif predictions[top_indices[2]] > threshold:
count_false_positives += 1
if top_indices[3] in correct_indices:
count_four_best += 1
num_correct += 1
if predictions[top_indices[3]] > threshold:
count_correct_positives += 1
elif predictions[top_indices[3]] > threshold:
count_false_positives += 1
if top_indices[4] in correct_indices:
count_five_best += 1
num_correct += 1
if predictions[top_indices[4]] > threshold:
count_correct_positives += 1
elif predictions[top_indices[4]] > threshold:
count_false_positives += 1
count_num_correct_in_top_5 += num_correct
else:
x_list, y_list = prepare_data_inference(data_dic,
num_samples=26)
for i in range(26):
x = x_list[i]
y = y_list[i]
curr_player = find_curr_player(x)
curr_player_hand_indices = range(curr_player * 32,
curr_player * 32 + 32)
predictions = model.predict(np.array([x]))[0]
top_indices = top_k_indices(predictions, k=5)
correct_indices = np.where(y == 1)[0]
num_correct = 0
num_correct_all_hands = 0
# Analyze general prediction rate for all hands
if top_indices[0] in correct_indices:
count_best_all_hands += 1
num_correct_all_hands += 1
if predictions[top_indices[0]] > threshold:
count_correct_positives_all_hands += 1
elif predictions[top_indices[0]] > threshold:
count_false_positives_all_hands += 1
if top_indices[1] in correct_indices:
count_two_all_hands += 1
num_correct_all_hands += 1
if predictions[top_indices[1]] > threshold:
count_correct_positives_all_hands += 1
elif predictions[top_indices[1]] > threshold:
count_false_positives_all_hands += 1
if top_indices[2] in correct_indices:
count_three_all_hands += 1
num_correct_all_hands += 1
if predictions[top_indices[2]] > threshold:
count_correct_positives_all_hands += 1
elif predictions[top_indices[2]] > threshold:
count_false_positives_all_hands += 1
if top_indices[3] in correct_indices:
count_four_all_hands += 1
num_correct_all_hands += 1
if predictions[top_indices[3]] > threshold:
count_correct_positives_all_hands += 1
elif predictions[top_indices[3]] > threshold:
count_false_positives_all_hands += 1
if top_indices[4] in correct_indices:
count_five_all_hands += 1
num_correct_all_hands += 1
if predictions[top_indices[4]] > threshold:
count_correct_positives_all_hands += 1
elif predictions[top_indices[4]] > threshold:
count_false_positives_all_hands += 1
count_num_correct_in_top_5_all_hands += num_correct_all_hands
# analyze opponent hand predictions only
predictions = model.predict(np.array([x]))[0]
predictions[curr_player_hand_indices] = 0
top_indices = top_k_indices(predictions, k=5)
if top_indices[0] in correct_indices and top_indices[
0] not in curr_player_hand_indices:
count_best_card += 1
num_correct += 1
if predictions[top_indices[0]] > threshold:
count_correct_positives += 1
elif predictions[top_indices[0]] > threshold:
count_false_positives += 1
if top_indices[1] in correct_indices and top_indices[
1] not in curr_player_hand_indices:
count_two_best += 1
num_correct += 1
if predictions[top_indices[1]] > threshold:
count_correct_positives += 1
elif predictions[top_indices[1]] > threshold:
count_false_positives += 1
if top_indices[2] in correct_indices and top_indices[
2] not in curr_player_hand_indices:
count_three_best += 1
num_correct += 1
if predictions[top_indices[2]] > threshold:
count_correct_positives += 1
elif predictions[top_indices[2]] > threshold:
count_false_positives += 1
if top_indices[3] in correct_indices and top_indices[
3] not in curr_player_hand_indices:
count_four_best += 1
num_correct += 1
if predictions[top_indices[3]] > threshold:
count_correct_positives += 1
elif predictions[top_indices[3]] > threshold:
count_false_positives += 1
if top_indices[4] in correct_indices and top_indices[
4] not in curr_player_hand_indices:
count_five_best += 1
num_correct += 1
if predictions[top_indices[4]] > threshold:
count_correct_positives += 1
elif predictions[top_indices[4]] > threshold:
count_false_positives += 1
count_num_correct_in_top_5 += num_correct
if not extended_model:
print('Analysis all hands : ')
print(count_best_all_hands, ' / ', num_games * 26,
' Predicted best card in : ',
count_best_all_hands / (num_games * 26))
print(count_two_all_hands, ' / ', num_games * 26,
' Predicted sec card in : ',
count_two_all_hands / (num_games * 26))
print(count_three_all_hands, ' / ', num_games * 26,
' Predicted third card in : ',
count_three_all_hands / (num_games * 26))
print(count_four_all_hands, ' / ', num_games * 26,
' Predicted fourth card in : ',
count_four_all_hands / (num_games * 26))
print(count_five_all_hands, ' / ', num_games * 26,
' Predicted fifth card in : ',
count_five_all_hands / (num_games * 26))
print('Average number of correct predictions: ',
count_num_correct_in_top_5_all_hands / (num_games * 26))
print('Bigger then threshold {} : {} / {} correct, {}'.format(
threshold, count_correct_positives_all_hands,
count_correct_positives_all_hands +
count_false_positives_all_hands,
count_correct_positives_all_hands /
(count_correct_positives_all_hands +
count_false_positives_all_hands)))
print('Analysis only opponent hands : ')
print(count_best_card, ' / ', num_games * 26, ' Predicted best card in : ',
count_best_card / (num_games * 26))
print(count_two_best, ' / ', num_games * 26, ' Predicted sec card in : ',
count_two_best / (num_games * 26))
print(count_three_best, ' / ', num_games * 26,
' Predicted third card in : ', count_three_best / (num_games * 26))
print(count_four_best, ' / ', num_games * 26,
' Predicted fourth card in : ', count_four_best / (num_games * 26))
print(count_five_best, ' / ', num_games * 26,
' Predicted fifth card in : ', count_five_best / (num_games * 26))
print('Average number of correct predictions: ',
count_num_correct_in_top_5 / (num_games * 26))
print('Bigger then threshold {} : {} / {} correct, {}'.format(
threshold, count_correct_positives,
count_correct_positives + count_false_positives,
count_correct_positives /
(count_correct_positives + count_false_positives)))
return count_correct_positives / (count_correct_positives +
count_false_positives)