def get_train_data(self) -> Tuple[np.array, np.array]:
""" Get the train data useable for machine learning algorithms.
Returns:
The train input and train output. The train input is a 2d array where each row represents a different train
element. The train output is 1d array of labels, where a 1 means that this player was the 'Mol' and a 0
means that this player was not the 'Mol'.
"""
self.__encoder = MoneyEncoder(self.__spline_clusters,
self.__num_other_money_quantiles,
self.__random_generator)
samples = self.__encoder.get_money_samples(self.__train_seasons,
math.inf)
self.__major_pattern = self.__encoder.major_money_pattern(samples)
self.__minor_pattern = self.__encoder.minor_money_pattern(samples)
train_input = np.array([
self.__encoder.extract_features(sample, self.__major_pattern,
self.__minor_pattern)
for sample in samples
])
train_output = np.array(
[1.0 if get_is_mol(sample.player) else 0.0 for sample in samples])
self.__pca = PCA(n_components=self.__pca_explain)
train_input = self.__pca.fit_transform(train_input)
return train_input, train_output
def get_train_data(self) -> Tuple[np.array, np.array]:
""" Get the train data useable for machine learning algorithms.
Returns:
The train input and train output. The train input is a 2d array where each row
represents a different train element. The train output is 1d array of labels, where a 1 means that this
player was the 'Mol' and a 0 means that this player was not the 'Mol'.
"""
train_data = []
for season in self.__train_seasons:
parsed_videos = self.__get_parsed_videos(season)
player_episodes = self.__get_players_with_episodes(
season, parsed_videos)
for player, episodes in player_episodes.items():
for episode in episodes:
# Do the Data Augmentation by turning cuts on/off.
for selection in itertools.product(
[False, True], repeat=self.__aug_num_cuts):
if sum(selection) >= self.__aug_min_cuts_on:
relative_occurrence = self.get_relative_occurrence(
player, parsed_videos[episode], selection)
train_data.append(
TrainSample(season, relative_occurrence,
get_is_mol(player)))
train_data = self.__filter_outliers(train_data)
train_input = np.array([[ts.relative_occurrence] for ts in train_data])
train_output = np.array(
[1.0 if ts.is_mol else 0.0 for ts in train_data])
return train_input, train_output
def __get_train_data(self, train_seasons: Set[int], alives_group: Set[int]) -> Tuple[np.array, np.array]:
""" Get the train data used to train the stacking algorithm.
Arguments:
train_seasons (Set[int]): All seasons used for training the stacking algorithm.
alives_group (Set[int]): Only episodes with any of these numbers of players that could still potentially
be the mol are used as training data.
Returns:
The train input which is a 2d matrix where each row represents a train case. Also it returns the train
output which is a 1d array where each value indicates whether the corresponding train case row was the mol
or not.
"""
train_input = []
train_output = []
for train_season in train_seasons:
for episode in range(get_last_episode(train_season) + 1):
train_prediction = self.__train_layer.predict(train_season, episode,
train_seasons.difference({train_season}))
train_alive = {player for player, result in train_prediction.items() if not result.exclusion}
if len(train_alive) in alives_group:
for player, result in train_prediction.items():
if not result.exclusion:
train_input.append(self.__input_encoding(result, len(train_alive)))
train_output.append(1.0 if get_is_mol(player) else 0.0)
return np.array(train_input), np.array(train_output)
def train(self):
""" Execute the training process for the Wikipedia Extractor. """
raw_train_data = WikipediaParser.parse(self.__train_seasons)
train_output = np.array([1.0 if get_is_mol(player) else 0.0 for player in raw_train_data])
job_input = np.array([data.job_features for data in raw_train_data.values()])
self.__train_job_clusters(job_input)
job_input = self.__discretize_jobs(job_input)
self.__pca = PCA(n_components = self.__pca_components)
job_input = self.__pca.fit_transform(job_input)
word_input = np.array([[data.word_feature ** i for i in range(1, self.WORD_COUNT_POLY_DEGREE + 1)]
for data in raw_train_data.values()])
train_input = np.concatenate((job_input, word_input), axis = 1)
self.__lda = LinearDiscriminantAnalysis()
self.__lda.fit(train_input, train_output)
def major_money_pattern(
self, samples: List[MoneySample]) -> Callable[[float], float]:
""" Learn the likelihood function how likely someone is the Mol based on what that player major earned during a
single exercise.
Arguments:
samples (List[MoneySample): The samples used to train the likelihood function
Returns:
A function that returns based on how many money that player major earned how likely that player is the Mol.
"""
train_input = np.array([[sample.major_self] for sample in samples])
train_output = np.array(
[1.0 if get_is_mol(sample.player) else 0.0 for sample in samples])
return self.__learn_money_pattern(train_input, train_output)
def __training(self, train_seasons: Set[int]) -> Tuple[gaussian_kde, gaussian_kde]:
""" Execute the training phase of the Age Layer.
Arguments:
train_seasons (Set[int]): The season used as training data
Returns:
The kernel density estimator for respectively the Mol data and non-Mol data.
"""
players = [player for player in Player if get_season(player) in train_seasons]
train_input = [float(get_age(player)) for player in players]
train_output = [1.0 if get_is_mol(player) else 0.0 for player in players]
non_mol = np.array([data for data, label in zip(train_input, train_output) if label == 0.0])
mol = np.array([data for data, label in zip(train_input, train_output) if label == 1.0])
non_mol_kde = InnerAppearanceLayer.kernel_density_estimation(non_mol)
mol_kde = InnerAppearanceLayer.kernel_density_estimation(mol)
return non_mol_kde, mol_kde
def log_loss(
distributions: Dict[Tuple[int, int], Dict[Player,
float]]) -> float:
""" Compute the log loss over the predictions.
Arguments:
distributions (Dict[Tuple[int, int], Dict[Player, float]]): All the predictions.
Returns:
The log loss over the predictions.
"""
log_loss = 0.0
num_pred = 0
for distribution in distributions.values():
for player, prob in distribution.items():
log_loss -= math.log(prob) if get_is_mol(player) else math.log(
1 - prob)
num_pred += 1
return log_loss / num_pred
def __concordant_discordant_ratio(
self, distributions: Dict[Tuple[int, int], Dict[Player,
float]]) -> float:
""" Compute the concordant-discordant ratio over the predictions.
Arguments:
distributions (Dict[Tuple[int, int], Dict[Player, float]]): All the predictions.
Returns:
The concordant discordant ratio.
"""
non_mol_likelihoods = []
mol_likelihoods = []
for id, distribution in distributions.items():
s, e = id
num_potential_mol = self.__num_potential_mol(s, e)
for player, prob in distribution.items():
if get_is_mol(player):
mol_likelihoods.append(num_potential_mol * prob)
else:
non_mol_likelihoods.append(num_potential_mol * prob)
concordant = 0
discordant = 0
for mol_likelihood, non_mol_likelihood in it.product(
mol_likelihoods, non_mol_likelihoods):
if math.isclose(mol_likelihood,
non_mol_likelihood,
abs_tol=self.FLOAT_DIFF_EQUALITY):
continue
if mol_likelihood > non_mol_likelihood:
concordant += 1
else:
discordant += 1
if concordant + discordant == 0:
return 1 / 2
else:
return concordant / (concordant + discordant)
def get_train_data(self) -> Tuple[np.array, np.array, np.array, np.array]:
""" Get the formatted and sampled train data with train weights useable for machine learning algorithms.
Returns:
The train input, train output, answered_on and train weights in this order. The train input is a 2d array
where each row represents a different train element. The train output is 1d array of labels, such that the
ith row of the train input corresponds to the ith element of the train output. The answered on is a 1d array
of binary values where a 1 indicates if the selected player is included in the answer and 0 otherwise. The
train weights is an array of weights indicating how important every train element is.
"""
train_data = []
for season in self.__train_seasons:
train_data.extend(self.get_season_data(season, sys.maxsize, True))
train_input = np.array([
ExamDropEncoder.extract_features(sample, sys.maxsize)
for sample in train_data
])
train_output = np.array([
1.0 if get_is_mol(sample.selected_player) else 0.0
for sample in train_data
])
answered_on = np.array([
1.0 if sample.selected_player in sample.answer else 0.0
for sample in train_data
])
m = ExamDropEncoder.NUM_CONTINUOUS_FEATURES
self.__discretizer = StableDiscretizer(self.__min_cluster_size,
self.__random_generator)
discrete_input = self.__discretizer.fit_transform(train_input[:, :-m])
self.__spline_encoder = NaturalSplineEncoding(
[self.__spline_curves for _ in range(m)])
continuous_input = self.__spline_encoder.fit_transform(
train_input[:, -m:])
train_input = np.concatenate((discrete_input, continuous_input),
axis=1)
return train_input, train_output, answered_on, self.__get_train_weights(
train_data)
def get_train_data(self) -> Tuple[np.array, np.array, np.array]:
""" Get the formatted and sampled train data with train weights useable for machine learning algorithms.
Returns:
The train input, train output and train weights in this order. The train input is a 2d array where each row
represents a different train element. The train output is 1d array of labels, such that the ith row of the
train input corresponds to the ith element of the train output.
"""
train_data = []
for season in self.__train_seasons:
train_data.extend(self.__get_season_data(season, sys.maxsize, True))
train_input = np.array([ExamDropEncoder.extract_features(sample, sys.maxsize) for sample in train_data])
train_output = np.array([1.0 if get_is_mol(sample.selected_player) else 0.0 for sample in train_data])
num_bins = self.get_num_bins(train_input, self.__max_splits)
self.__discretizer = KBinsDiscretizer(n_bins = num_bins, encode = "onehot-dense",
strategy = ExamDropExtractor.BIN_STRATEGY)
train_input = self.__discretizer.fit_transform(train_input)
train_input = self.__add_answered_on_feature(train_data, train_input)
self.__anova_f_filter = SelectFpr(f_classif, alpha = self.__anova_f_significance)
train_input = self.__anova_f_filter.fit_transform(train_input, train_output)
self.__pca = PCA(n_components = self.__pca_explain)
train_input = self.__pca.fit_transform(train_input)
return train_input, train_output, self.__get_train_weights(train_data)
progress_bar = Bar("Distributions Computed:", max = len(SEASONS))
non_mol_outliers = 0
mol_outliers = 0
non_mol_inliers = 0
mol_inliers = 0
for season in SEASONS:
random_generator = RandomState(RANDOM_SEED)
train_seasons = SEASONS.difference({season})
extractor = WikipediaExtractor(season, train_seasons, PCA_COMPONENTS, UNLIKELY_Z_SCORE, random_generator)
extractor.train()
predict_input = extractor.get_predict_data()
for player, is_outlier in predict_input.items():
if is_outlier[0]:
if get_is_mol(player):
mol_outliers += 1
else:
non_mol_outliers += 1
else:
if get_is_mol(player):
mol_inliers += 1
else:
non_mol_inliers += 1
progress_bar.next()
inlier_likelihood = mol_inliers / (non_mol_inliers + mol_inliers)
outlier_likelihood = mol_outliers / (non_mol_outliers + mol_outliers)
lower_likelihood = outlier_likelihood / inlier_likelihood
progress_bar.finish()
from Layers.Appearance.AppearanceExtractor import AppearanceExtractor
from Layers.Appearance.VideoParser import VideoParser, ParsedVideo
from scipy.stats import pearsonr, kendalltau
import itertools
TEST_SEASONS = {13, 14, 15, 16, 17, 18, 19, 20}
appearances = dict()
for season in TEST_SEASONS:
for episode in itertools.count(1):
parsed_video = VideoParser.load_parsed_video(season, episode)
if parsed_video is None:
break
for player in parsed_video.alive_players:
if not get_is_mol(player):
appearance = AppearanceExtractor.get_relative_occurrence(player, parsed_video, [True])
appearances[player] = appearances.get(player, []) + [appearance]
input = []
output = []
for player, features in appearances.items():
for feat1, feat2 in itertools.combinations(features, 2):
input.append(feat1)
output.append(feat2)
r, p_value = pearsonr(input, output)
print("Pearson Test (Between):")
print("R value: " + str(r))
print("R-squared value: " + str(r ** 2))
print("p-value: " + str(p_value))
TRAIN_SEASONS = {
5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22
}
MIN_CLUSTER_SIZE = 120
NUM_CURVES = 4
train_data = []
for season in TRAIN_SEASONS:
train_data.extend(
ExamDropExtractor.get_season_data(season, sys.maxsize, True))
train_input = np.array([
ExamDropEncoder.extract_features(sample, sys.maxsize)
for sample in train_data
])
train_output = np.array([
1.0 if get_is_mol(sample.selected_player) else 0.0 for sample in train_data
])
m = ExamDropEncoder.NUM_CONTINUOUS_FEATURES
discretizer = StableDiscretizer(MIN_CLUSTER_SIZE)
discrete_input = discretizer.fit_transform(train_input[:, :-m])
spline_encoder = NaturalSplineEncoding([NUM_CURVES for _ in range(m)])
continuous_input = spline_encoder.fit_transform(train_input[:, -m:])
trans_input = np.concatenate((discrete_input, continuous_input), axis=1)
in_answer_input = [
row for row, data in zip(trans_input, train_data)
if data.selected_player in data.answer
]
in_answer_output = [
to for to, data in zip(train_output, train_data)
value for value, is_mol in zip(train_input, train_output) if is_mol == 1.0
]
non_mol_features = [
value for value, is_mol in zip(train_input, train_output) if is_mol == 0.0
]
_, mean_p_value = kruskal(mol_features, non_mol_features)
_, std_p_value = levene(mol_features, non_mol_features)
print("Mean: " + str(mean_p_value) + ", Std: " + str(std_p_value))
mol_kde = InnerAppearanceLayer.kernel_density_estimation(mol_features)
non_mol_kde = InnerAppearanceLayer.kernel_density_estimation(non_mol_features)
predict_data = extractor.get_predict_data()
predict_input = np.array([data for data in predict_data.values()])
predict_output = np.array(
[1.0 if get_is_mol(player) else 0.0 for player in predict_data.keys()])
mean, std = norm.fit(predict_input)
z_cutoff = mean + std * LOWER_Z_SCORE
train_output += np.random.uniform(-0.1, 0.1, len(train_output))
predict_output += np.random.uniform(-0.1, 0.1, len(predict_output))
plt.figure(figsize=(12, 3))
plt.xlabel("Job Feature")
plt.ylabel("Is 'mol'")
plt.yticks([0.0, 1.0])
plt.gcf().subplots_adjust(bottom=0.15)
# plt.plot(X, mol_y, color = "r")
# plt.plot(X, non_mol_y, color = "g")
# plt.plot(X, posterior, color = "b")
plt.axvline(x=z_cutoff, c='black')
def compute_distribution(self, predict_season: int, latest_episode: int, train_seasons: Set[int]) -> Dict[Player, float]:
return {player: 1.0 if get_is_mol(player) else 0.0 for player in get_players_in_season(predict_season)}
import numpy as np
import sys
TRAIN_SEASONS = {
5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22
}
train_data = []
for season in TRAIN_SEASONS:
train_data.extend(
ExamDropExtractor.get_season_data(season, sys.maxsize, True))
train_input = np.array([
ExamDropEncoder.extract_features(sample, sys.maxsize)
for sample in train_data
])
in_answer_ratio = sum(1 for data, input in zip(train_data, train_input) if data.selected_player in data.answer) \
/ len(train_data)
is_mol_ratio = sum(1 for data, input in zip(train_data, train_input) if get_is_mol(data.selected_player)) \
/ len(train_data)
both_ratio = sum(1 for data, input in zip(train_data, train_input)
if get_is_mol(data.selected_player)
and data.selected_player in data.answer) / len(train_data)
print("In answer ratio: " + str(in_answer_ratio))
print("Is mol ratio: " + str(is_mol_ratio))
print("Both ratio: " + str(both_ratio))
print()
for name, column in zip(ExamDropEncoder.FEATURE_NAMES, train_input.T):
counter = Counter(column)
print(name + ": ", [(value, counter[value]) for value in sorted(counter)])
from Layers.Special.MemoryLayer import MemoryLayer
from scipy.stats import pearsonr, kendalltau
import math
TRAIN_SEASONS = {
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21
}
TEST_SEASONS = {9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
layer1 = MemoryLayer("Wikipedia Stacker")
layer2 = MemoryLayer("Appearance Stacker")
pairs = []
for season in TEST_SEASONS:
players = {
player
for player in get_players_in_season(season) if not get_is_mol(player)
}
for episode in range(get_last_episode(season) + 1):
prediction1 = layer1.compute_distribution(season, episode,
TRAIN_SEASONS)
prediction2 = layer2.compute_distribution(season, episode,
TRAIN_SEASONS)
excluded = {
player
for player, prob in prediction1.items() if prob == 0.0
}
excluded.update(
{player
for player, prob in prediction2.items() if prob == 0.0})
included = players.difference(excluded)
from Layers.ExamDrop.ExamDropExtractor import ExamDropExtractor
from Tools.Encoders.NaturalSplineEncoding import NaturalSplineEncoding
from Tools.Encoders.StableDiscretizer import StableDiscretizer
from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt
import numpy as np
import sys
TRAIN_SEASONS = {5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22}
NUM_CURVES = 4
train_data = []
for season in TRAIN_SEASONS:
train_data.extend(ExamDropExtractor.get_season_data(season, sys.maxsize, True))
train_input = np.array([ExamDropEncoder.extract_features(sample, sys.maxsize) for sample in train_data])
train_output = np.array([1.0 if get_is_mol(sample.selected_player) else 0.0 for sample in train_data])
m = ExamDropEncoder.NUM_CONTINUOUS_FEATURES
for column, feature_name in zip(train_input.T[-m:], ExamDropEncoder.FEATURE_NAMES[-m:]):
trans_data = column[:,np.newaxis]
spline_encoder = NaturalSplineEncoding([NUM_CURVES])
trans_data = spline_encoder.fit_transform(trans_data)
in_answer_input = [row for row, data in zip(trans_data, train_data) if data.selected_player in data.answer]
in_answer_output = [to for to, data in zip(train_output, train_data) if data.selected_player in data.answer]
regression = LinearRegression()
regression.fit(in_answer_input, in_answer_output)
X = np.array([pi for pi in sorted(set(column))])
predict_input = spline_encoder.transform(X[:, np.newaxis])
predict_output = regression.predict(predict_input)
r = 1 / (2 * math.sqrt(math.pi))
non_mol_term = non_mol_kde.pdf(x)[0] / (non_mol_points *
math.sqrt(non_mol_bandwidth))
mol_term = mol_kde.pdf(x)[0] / (mol_points * math.sqrt(mol_bandwidth))
sigma = (r * (non_mol_term + mol_term))**2
return u / sigma
TEST_SEASONS = {
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21
}
ALPHA = 0.05
players = [player for player in Player if get_season(player) in TEST_SEASONS]
train_input = [float(get_age(player)) for player in players]
train_output = [1.0 if get_is_mol(player) else 0.0 for player in players]
non_mol = np.array(
[data for data, label in zip(train_input, train_output) if label == 0.0])
mol = np.array(
[data for data, label in zip(train_input, train_output) if label == 1.0])
non_mol_kde = InnerAppearanceLayer.kernel_density_estimation(non_mol)
non_mol_bandwidth = silverman_bandwidth(np.array(non_mol))
non_mol_points = len(non_mol)
mol_kde = InnerAppearanceLayer.kernel_density_estimation(mol)
mol_bandwidth = silverman_bandwidth(np.array(mol))
mol_points = len(mol)
ages = [float(age) for age in range(20, 59)]
statistic_values = [
test_statistic(age, non_mol_kde, non_mol_bandwidth, non_mol_points,
from Data.PlayerData import get_is_mol
from Data.Wikipedia.Job import Job
from Layers.Wikipedia.WikipediaParser import WikipediaParser
from scipy.stats import kruskal, levene
import numpy as np
TRAIN_SEASONS = {5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}
data = WikipediaParser.parse(TRAIN_SEASONS)
predict_input = np.array([d.job_features for p, d in data.items()])
predict_output = np.array([1.0 if get_is_mol(p) else 0.0 for p in data])
for job, column in zip(Job, predict_input.T):
mol_features = [
value for value, is_mol in zip(column, predict_output) if is_mol == 1.0
]
non_mol_features = [
value for value, is_mol in zip(column, predict_output) if is_mol == 0.0
]
_, mean_p_value = kruskal(mol_features, non_mol_features)
_, std_p_value = levene(mol_features, non_mol_features)
print(
str(job) + " - Mean: " + str(mean_p_value) + ", Std: " +
str(std_p_value))
from Data.PlayerData import get_is_mol
from Data.Wikipedia.Job import Job
from Layers.Wikipedia.WikipediaParser import WikipediaParser
import matplotlib.pyplot as plt
SEASONS = {5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}
JOB = Job.TV_PRODUCER
data = WikipediaParser.parse(SEASONS)
job_mapping = dict([(job, i) for i, job in enumerate(Job)])
job_index = job_mapping[JOB]
train_input = [d.job_features[job_index] for p, d in data.items()]
train_output = [1.0 if get_is_mol(p) else 0.0 for p in data]
plt.figure(figsize=(12, 3))
plt.xlabel("Score")
plt.ylabel("Is 'mol'")
plt.yticks([0.0, 1.0])
plt.gcf().subplots_adjust(bottom=0.15)
plt.scatter(train_input, train_output, s=4)
plt.show()
for season in TEST_SEASONS:
for episode in itertools.count(1):
parsed_video = VideoParser.load_parsed_video(season, episode)
if parsed_video is None:
break
for player in parsed_video.alive_players:
appearance = AppearanceExtractor.get_relative_occurrence(
player, parsed_video, [True])
appearances[player] = appearances.get(player, []) + [appearance]
train_input = []
train_output = []
for player, features in appearances.items():
train_input.append(mean(features))
train_output.append(1.0 if get_is_mol(player) else 0.0)
non_mol = [
data for data, label in zip(train_input, train_output) if label == 0.0
]
mol = [data for data, label in zip(train_input, train_output) if label == 1.0]
statistics, p_value = mannwhitneyu(mol, non_mol, alternative="less")
print("Score: " + str(statistics))
print("p-value: " + str(p_value))
train_output += np.random.uniform(-0.1, 0.1, len(train_output))
print("Number of non-Mol players: " + str(len(non_mol)))
print("Number of Mol players: " + str(len(mol)))
plt.figure(figsize=(12, 3))
plt.scatter(train_input, train_output, s=4)
plt.xlabel("Relative Appearance")
