def plot_distance_against_fpr(network):
network = network.eval()
distances = list()
with open('named_firstpickrates.csv', 'r') as f:
reader = csv.reader(f, delimiter=';')
lines = [line for line in reader]
indizes = [i for i in range(265)]
pickrates = [float(lines[index][1]) for index in indizes]
with torch.no_grad():
empty = torch.zeros(1, 265)
anchor = network(empty)
for card in range(265):
tmp = torch.zeros((1, 265))
tmp[0, card] = 1
output = network(tmp)
distances.append(networks.get_distance(anchor, output).item())
elements = [(pickrates[i], distances[i]) for i in range(265)]
elements.sort(key=lambda v: v[0])
plt.scatter([element[0] for element in elements],
[element[1] for element in elements],
alpha=0.6)
plt.xlabel('First-pick rate')
plt.ylabel('Distance to empty set')
plt.savefig('plots/fpr_distance_2.pdf', bbox_inches='tight')
def evaluate_preferences(dataset, net):
comparisons = 0
correct_comparisons = 0
net.eval()
for anchor, positive, negative in dataset:
if comparisons == 100000:
break
comparisons += anchor.shape[0]
out1 = net(anchor)
out2 = net(positive)
out3 = net(negative)
for i in range(anchor.shape[0]):
if models.get_distance(out1[i, :].view(1, -1), out2[i, :].view(
1, -1)) < models.get_distance(out1[i, :].view(1, -1),
out3[i, :].view(1, -1)):
correct_comparisons += 1
return correct_comparisons / comparisons
def correct_picks(network, folder_name):
pick_number = 0
correct_picks = 0
pick_distance = list()
pick_numbers = list()
files = os.listdir(folder_name)
with open(folder_name + np.random.choice(files), 'rb') as f:
data = np.load(f, allow_pickle=True)
for pick in data[
np.random.choice(data.shape[0], 10000, replace=False), :]:
pick_number += 1
choices = pick[0]
pick_numbers.append(len(choices))
anchor = pick[1]
correct_choice = np.argmax(pick[2])
with torch.no_grad():
out1 = network(torch.Tensor(anchor).view(1, -1))
distances = list()
for i in range(len(choices)):
elem = torch.zeros((1, 265))
elem[0, choices[i]] = 1.0
distances.append(
networks.get_distance(out1, network(elem)).item())
ranking = [
x for _, x in sorted(
zip(distances, [y for y in range(len(distances))]))
]
if choices[ranking[0]] == correct_choice:
correct_picks += 1
pick_distance.append(0)
else:
pick_distance.append(
ranking.index(np.where(choices == correct_choice)[0]))
f.close()
del data
print(np.mean(pick_numbers))
return correct_picks / pick_number, np.mean(pick_distance)
def complete_draft(picks, network):
network.eval()
every_choice = list()
every_chosen = list()
every_anchor = list()
random_choices = list()
random_anchors = list()
siamese_distance = 0
random_distance = 0
with torch.no_grad():
picked = torch.zeros((1, 265))
random_picked = torch.zeros((1, 265))
for pick in picks:
anchor = network(picked)
random_anchor = network(random_picked)
every_anchor.append(anchor.squeeze().detach().numpy())
random_anchors.append(random_anchor.squeeze().detach().numpy())
choices = pick[0]
every_choice.append(choices)
distances = list()
for choice in choices:
tmp = torch.zeros((1, 265))
tmp[0, choice] = 1
distances.append(
networks.get_distance(anchor, network(tmp)).item())
chosen_card = choices[np.argmin(distances)]
siamese_distance += np.min(distances)
random_index = np.random.randint(0, len(choices))
randomly_chosen_card = choices[random_index]
indx = list(np.where(choices == randomly_chosen_card))[0]
random_distance += distances[random_index]
picked[0, chosen_card] += 1
random_picked[0, randomly_chosen_card] += 1
every_chosen.append(chosen_card)
random_choices.append(randomly_chosen_card)
every_anchor.append(anchor.squeeze().detach().numpy())
return every_choice, every_chosen, every_anchor, random_choices, random_anchors, siamese_distance, random_distance
def point_cloud(network, embedding=None):
network = network.eval()
card_dict = load_card_dict('card_dict.pt')
inv_card_dict = {v: k for k, v in card_dict.items()}
cards = list()
if embedding is not None:
tsne_embedding = embedding
embeddings = list()
labels = list()
colors = list()
distances = list()
with open('Data/named_pickrates.csv', 'r') as f:
reader = csv.reader(f, delimiter=';')
lines = [line for line in reader]
indizes = [i for i in range(265)]
pickrates = [float(lines[index][1]) for index in indizes]
with torch.no_grad():
empty = torch.zeros(1, 265)
anchor = network(empty)
distances.append(0)
embeddings.append(anchor.squeeze().numpy())
colors.append('lime')
border_colors = list()
border_colors.append('lime')
color_map = {
'Colourless': 'purple',
'Red': 'red',
'Blue': 'blue',
'Green': 'green',
'White': 'lightgrey',
'Black': 'black'
}
for card in range(265):
tmp = torch.zeros((1, 265))
tmp[0, card] = 1
output = network(tmp)
embeddings.append(output.squeeze().numpy())
distances.append(networks.get_distance(anchor, output).item())
c = lines[card][4]
c = c.split(',')
if len(c) == 1:
colors.append(color_map[c[0]])
border_colors.append(color_map[c[0]])
elif len(c) == 2:
colors.append(color_map[c[0]])
border_colors.append(color_map[c[1]])
else:
colors.append('gold')
border_colors.append('gold')
if embedding is None:
tsne_embedding = TSNE(n_components=2).fit_transform(embeddings)
tsne_distances = [
networks.get_distance(
torch.Tensor(tsne_embedding[0]).view(1, -1),
torch.Tensor(tsne_embedding[i]).view(1, -1))
for i in range(1, len(tsne_embedding))
]
max_distance = np.max(distances)
plt.scatter(tsne_embedding[:, 0],
tsne_embedding[:, 1],
c=colors,
edgecolors=border_colors)
plt.annotate('Anchor', (tsne_embedding[0, 0], tsne_embedding[0, 1]))
plt.savefig('plots/point_cloud.pdf', bbox_inches='tight')
plt.show()
return tsne_embedding
def plot_random_embeddings(network):
card_dict = load_card_dict('card_dict.pt')
inv_card_dict = {v: k for k, v in card_dict.items()}
cards = list()
for i in range(265):
tmp = np.zeros(265)
tmp[i] = 1
cards.append(torch.Tensor(tmp))
embeddings = list()
labels = list()
colors = list()
distances = list()
with open('named_pickrates.csv', 'r') as f:
reader = csv.reader(f, delimiter=';')
lines = [line for line in reader]
indizes = [i for i in range(265)]
pickrates = [float(lines[index][1]) for index in indizes]
with torch.no_grad():
empty = torch.zeros(1, 265)
anchor = network(empty)
embeddings.append(anchor.squeeze().numpy())
labels.append('Empty set')
colors.append('r')
for card in cards:
output = network(card)
embeddings.append(output.squeeze().numpy())
distances.append(networks.get_distance(anchor, output))
# labels.append(inv_card_dict[card.item()])
colors.append('b')
tsne_embedding = TSNE(n_components=2).fit_transform(embeddings)
tsne_distances = [
networks.get_distance(
torch.Tensor(tsne_embedding[0]).view(1, -1),
torch.Tensor(tsne_embedding[i]).view(1, -1))
for i in range(1, len(tsne_embedding))
]
plt.scatter(tsne_embedding[:, 0], tsne_embedding[:, 1], c=colors)
#for i in range(1,11):
# plt.plot([tsne_embedding[0][0],tsne_embedding[i][0]],[tsne_embedding[0][1],tsne_embedding[i][1]])
# plt.annotate(str(pickrates[i-1]),((tsne_embedding[i,0]+tsne_embedding[0,0])/2,(tsne_embedding[i,1]+tsne_embedding[0,1])/2))
#for i,label in enumerate(labels):
# plt.annotate(label, (tsne_embedding[i,0],tsne_embedding[i,1]))
print(distances)
print(pickrates)
plt.show()
pickrate_ranking = construct_ranking(pickrates, True)
print(pickrate_ranking)
distance_ranking = construct_ranking(distances, False)
print(distance_ranking)
tsne_distances_ranking = construct_ranking(tsne_distances, False)
print(tsne_distances_ranking)
print('Embedded correlation: ' +
str(sc.stats.kendalltau(pickrate_ranking, distance_ranking)[0]))
print(
'TSNE correlation: ' +
str(sc.stats.kendalltau(pickrate_ranking, tsne_distances_ranking)[0]))
return tsne_embedding
def point_cloud_anchors_distances(network, embedding=None):
network = network.eval()
card_dict = load_card_dict('card_dict.pt')
inv_card_dict = {v: k for k, v in card_dict.items()}
cards = list()
if embedding is not None:
tsne_embedding = embedding
embeddings = list()
labels = list()
random_colors = list()
deck_colors = list()
tensor_embeddings = list()
distances = list()
with open('named_pickrates.csv', 'r') as f:
reader = csv.reader(f, delimiter=';')
lines = [line for line in reader]
indizes = [i for i in range(265)]
pickrates = [float(lines[index][1]) for index in indizes]
with torch.no_grad():
empty = torch.zeros(1, 265)
anchor = network(empty)
distances.append(0)
embeddings.append(anchor.squeeze().numpy())
border_colors = list()
border_colors.append('lime')
color_map = {
'Colourless': 'purple',
'Red': 'red',
'Blue': 'blue',
'Green': 'green',
'White': 'beige',
'Black': 'black'
}
for card in range(265):
tmp = torch.zeros((1, 265))
tmp[0, card] = 1
output = network(tmp)
tensor_embeddings.append(output)
embeddings.append(output.squeeze().numpy())
distances.append(networks.get_distance(anchor, output).item())
for i in range(1):
tmp = torch.zeros((1, 265))
for _ in range(45):
tmp[0, np.random.randint(265)] = 1
out = network(tmp)
random_distances = [
networks.get_distance(out, elem).item()
for elem in tensor_embeddings
]
random_colors.extend(random_distances)
embeddings.append(out.squeeze().numpy())
for i in range(1):
with open(
'E:/picks_2/' +
np.random.choice(os.listdir('E:/picks_2/')),
'rb') as pick_file:
some_picks = np.load(pick_file, allow_pickle=True)
anchor = 0
while anchor != 44:
anchor_array = some_picks[np.random.randint(
some_picks.shape[0])]
anchor = sum(anchor_array[1])
input = torch.Tensor(anchor_array[1] + anchor_array[2])
out = network(input)
deck_distances = [
networks.get_distance(out, elem).item()
for elem in tensor_embeddings
]
deck_colors.extend(deck_distances)
embeddings.append(out.squeeze().numpy())
if embedding is None:
tsne_embedding = TSNE(n_components=2).fit_transform(embeddings)
plt.scatter(tsne_embedding[1:-2, 0],
tsne_embedding[1:-2, 1],
c=random_colors,
cmap=plt.cm.rainbow)
plt.scatter(tsne_embedding[0, 0], tsne_embedding[0, 1], c='lime')
plt.scatter(tsne_embedding[-2, 0], tsne_embedding[-2, 1], c='black')
plt.annotate('Anchor', (tsne_embedding[0, 0], tsne_embedding[0, 1]))
plt.colorbar()
plt.savefig('plots/point_cloud_sets_random.pdf', bbox_inches='tight')
plt.show()
plt.scatter(tsne_embedding[1:-2, 0],
tsne_embedding[1:-2, 1],
c=deck_colors,
cmap=plt.cm.rainbow)
plt.scatter(tsne_embedding[0, 0], tsne_embedding[0, 1], c='lime')
plt.scatter(tsne_embedding[-1, 0], tsne_embedding[-1, 1], c='black')
plt.annotate('Anchor', (tsne_embedding[0, 0], tsne_embedding[0, 1]))
plt.colorbar()
plt.savefig('plots/point_cloud_sets_deck.pdf', bbox_inches='tight')
plt.show()
return tsne_embedding
def point_cloud_anchors_colors(network, embedding=None):
network = network.eval()
card_dict = load_card_dict('card_dict.pt')
inv_card_dict = {v: k for k, v in card_dict.items()}
cards = list()
if embedding is not None:
tsne_embedding = embedding
embeddings = list()
labels = list()
colors = list()
distances = list()
with open('named_pickrates.csv', 'r') as f:
reader = csv.reader(f, delimiter=';')
lines = [line for line in reader]
indizes = [i for i in range(265)]
pickrates = [float(lines[index][1]) for index in indizes]
with torch.no_grad():
empty = torch.zeros(1, 265)
anchor = network(empty)
distances.append(0)
embeddings.append(anchor.squeeze().numpy())
colors.append('lime')
border_colors = list()
border_colors.append('lime')
color_map = {
'Colourless': 'purple',
'Red': 'red',
'Blue': 'blue',
'Green': 'green',
'White': 'beige',
'Black': 'black'
}
for card in range(265):
tmp = torch.zeros((1, 265))
tmp[0, card] = 1
output = network(tmp)
embeddings.append(output.squeeze().numpy())
distances.append(networks.get_distance(anchor, output).item())
colors.append('grey')
border_colors.append('black')
for i in range(50):
with open(
'E:/picks_2/' +
np.random.choice(os.listdir('E:/picks_2/')),
'rb') as pick_file:
some_picks = np.load(pick_file, allow_pickle=True)
anchor = 0
c = False
while anchor != 44 or not c:
anchor_array = some_picks[np.random.randint(
some_picks.shape[0])]
anchor = sum(anchor_array[1])
input = anchor_array[1] + anchor_array[2]
if anchor == 44:
c = matching_colors(lines, input, 'Red')
if c:
tmp = torch.Tensor(input)
embeddings.append(network(tmp).squeeze().numpy())
colors.append(color_map[c])
if embedding is None:
tsne_embedding = TSNE(n_components=2).fit_transform(embeddings)
tsne_distances = [
networks.get_distance(
torch.Tensor(tsne_embedding[0]).view(1, -1),
torch.Tensor(tsne_embedding[i]).view(1, -1))
for i in range(1, len(tsne_embedding))
]
max_distance = np.max(distances)
plt.scatter(tsne_embedding[:, 0], tsne_embedding[:, 1], c=colors)
plt.annotate('Anchor', (tsne_embedding[0, 0], tsne_embedding[0, 1]))
plt.savefig('plots/point_cloud_sets_colored.pdf', bbox_inches='tight')
plt.show()
return tsne_embedding
def point_cloud_rarity(network, embedding=None):
network = network.eval()
card_dict = load_card_dict('card_dict.pt')
inv_card_dict = {v: k for k, v in card_dict.items()}
cards = list()
if embedding is not None:
tsne_embedding = embedding
embeddings = list()
labels = list()
colors = list()
distances = list()
with open('named_pickrates.csv', 'r') as f:
reader = csv.reader(f, delimiter=';')
lines = [line for line in reader]
indizes = [i for i in range(265)]
pickrates = [float(lines[index][1]) for index in indizes]
with torch.no_grad():
empty = torch.zeros(1, 265)
anchor = network(empty)
distances.append(0)
embeddings.append(anchor.squeeze().numpy())
colors.append('lime')
border_colors = list()
border_colors.append('lime')
color_map = {'C': 'black', 'U': 'silver', 'R': 'gold', 'M': 'brown'}
for card in range(265):
tmp = torch.zeros((1, 265))
tmp[0, card] = 1
output = network(tmp)
embeddings.append(output.squeeze().numpy())
distances.append(networks.get_distance(anchor, output).item())
c = lines[card][3]
c = c.split(',')
if len(c) == 1:
colors.append(color_map[c[0]])
border_colors.append(color_map[c[0]])
elif len(c) == 2:
colors.append(color_map[c[0]])
border_colors.append(color_map[c[1]])
else:
colors.append('gold')
border_colors.append('gold')
if embedding is None:
tsne_embedding = TSNE(n_components=2).fit_transform(embeddings)
tsne_distances = [
networks.get_distance(
torch.Tensor(tsne_embedding[0]).view(1, -1),
torch.Tensor(tsne_embedding[i]).view(1, -1))
for i in range(1, len(tsne_embedding))
]
max_distance = np.max(distances)
plt.scatter(tsne_embedding[:, 0],
tsne_embedding[:, 1],
c=colors,
edgecolors=border_colors)
plt.annotate('Anchor', (tsne_embedding[0, 0], tsne_embedding[0, 1]))
with open('plot_understanding.csv', 'w') as tmp:
writer = csv.writer(tmp)
for i in range(len(tsne_embedding) - 1):
lines[i].extend(tsne_embedding[i + 1][:])
writer.writerow(lines[i])
rank = construct_ranking(distances, reverse=False)
for card in rank[1:]:
print(inv_card_dict[card - 1])
print(np.argmax(pickrates))
plt.savefig('plots/point_cloud_rarity.pdf', bbox_inches='tight')
plt.show()
return tsne_embedding
def plot_accuracy_per_pick(network):
with open('exact_correct.tsv') as f:
reader = csv.reader(f, delimiter="\t")
lines = [line for line in reader]
accuracies = [[list() for _ in range(45)] for __ in range(5)]
for line in lines[1:]:
for i, elem in enumerate(line[3:]):
accuracies[i][int(line[1]) - 1].append(int(elem))
pick_number = [0 for _ in range(45)]
correct_picks = [0 for _ in range(45)]
pick_distance = list()
pick_numbers = list()
folder_name = 'E:/picks_2/'
with open(folder_name + np.random.choice(os.listdir(folder_name)),
'rb') as f:
data = np.load(f, allow_pickle=True)
for pick in data:
chosen_num = int(sum(pick[1]))
pick_number[chosen_num] += 1
choices = pick[0]
pick_numbers.append(len(choices))
anchor = pick[1]
correct_choice = np.argmax(pick[2])
with torch.no_grad():
out1 = network(torch.Tensor(anchor).view(1, -1))
distances = list()
for i in range(len(choices)):
elem = torch.zeros((1, 265))
elem[0, choices[i]] = 1.0
distances.append(
networks.get_distance(out1, network(elem)).item())
ranking = [
x for _, x in sorted(
zip(distances, [y for y in range(len(distances))]))
]
if choices[ranking[0]] == correct_choice:
correct_picks[chosen_num] += 1
f.close()
del data
plt.plot([j for j in range(15)],
[correct_picks[i] / pick_number[i] for i in range(15)],
c='lightslategrey',
label='SiameseBot')
plt.plot([j for j in range(15, 30)],
[correct_picks[i] / pick_number[i] for i in range(15, 30)],
c='lightslategrey')
plt.plot([j for j in range(30, 45)],
[correct_picks[i] / pick_number[i] for i in range(30, 45)],
c='lightslategrey')
plt.plot(
[j for j in range(15)],
[sum(accuracies[0][i]) / len(accuracies[0][i]) for i in range(15)],
c='red',
label='RandomBot')
plt.plot(
[j for j in range(15, 30)],
[sum(accuracies[0][i]) / len(accuracies[0][i]) for i in range(15, 30)],
c='red')
plt.plot(
[j for j in range(30, 45)],
[sum(accuracies[0][i]) / len(accuracies[0][i]) for i in range(30, 45)],
c='red')
plt.plot(
[j for j in range(15)],
[sum(accuracies[1][i]) / len(accuracies[1][i]) for i in range(15)],
c='blue',
label='RaredraftBot')
plt.plot(
[j for j in range(15, 30)],
[sum(accuracies[1][i]) / len(accuracies[1][i]) for i in range(15, 30)],
c='blue')
plt.plot(
[j for j in range(30, 45)],
[sum(accuracies[1][i]) / len(accuracies[1][i]) for i in range(30, 45)],
c='blue')
plt.plot(
[j for j in range(15)],
[sum(accuracies[2][i]) / len(accuracies[2][i]) for i in range(15)],
c='green',
label='DraftsimBot')
plt.plot(
[j for j in range(15, 30)],
[sum(accuracies[2][i]) / len(accuracies[2][i]) for i in range(15, 30)],
c='green')
plt.plot(
[j for j in range(30, 45)],
[sum(accuracies[2][i]) / len(accuracies[2][i]) for i in range(30, 45)],
c='green')
plt.plot(
[j for j in range(15)],
[sum(accuracies[3][i]) / len(accuracies[3][i]) for i in range(15)],
c='purple',
label='BayesBot')
plt.plot(
[j for j in range(15, 30)],
[sum(accuracies[3][i]) / len(accuracies[3][i]) for i in range(15, 30)],
c='purple')
plt.plot(
[j for j in range(30, 45)],
[sum(accuracies[3][i]) / len(accuracies[3][i]) for i in range(30, 45)],
c='purple')
plt.plot(
[j for j in range(15)],
[sum(accuracies[4][i]) / len(accuracies[4][i]) for i in range(15)],
c='orange',
label='NNetBot')
plt.plot(
[j for j in range(15, 30)],
[sum(accuracies[4][i]) / len(accuracies[4][i]) for i in range(15, 30)],
c='orange')
plt.plot(
[j for j in range(30, 45)],
[sum(accuracies[4][i]) / len(accuracies[4][i]) for i in range(30, 45)],
c='orange')
plt.xlim(0, 45)
plt.ylim(0, 1)
plt.xlabel('Pick')
plt.ylabel('Accuracy')
print(sum(correct_picks) / sum(pick_number))
plt.legend(ncol=2, loc=4)
plt.savefig('plots/accuracy_per_pick_comp.pdf', bbox_inches='tight')
plt.show()