def test_earliest_date_time_without_results(self):
FullMatchFactory(
start_date_time=timezone.localtime() - timedelta(days=1),
home_team_match__score=50,
away_team_match__score=80,
)
FullMatchFactory(
start_date_time=timezone.localtime() + timedelta(days=1),
home_team_match__score=0,
away_team_match__score=0,
)
with self.subTest("when all matches have results or haven't been played"):
earliest_date_time_without_results = (
Match.earliest_date_time_without_results()
)
self.assertIsNone(earliest_date_time_without_results)
played_resultless = FullMatchFactory(
start_date_time=timezone.localtime() - timedelta(days=1),
home_team_match__score=0,
away_team_match__score=0,
)
earliest_date_time_without_results = Match.earliest_date_time_without_results()
self.assertEqual(
played_resultless.start_date_time, earliest_date_time_without_results
)
def setUp(self):
self.maxDiff = None
self.client = Client(schema)
self.ml_models = [
MLModelFactory(
name=model_name,
is_principal=(idx == 0),
used_in_competitions=True,
# pylint: disable=unsubscriptable-object
prediction_type=PredictionType.values[idx],
) for idx, model_name in enumerate(MODEL_NAMES)
]
self.matches = [
FullMatchFactory(
with_predictions=True,
year=year,
round_number=((round_n % 23) + 1),
start_date_time=timezone.make_aware(
datetime(year, 6, (round_n % 29) + 1, match_n * 5)),
prediction__ml_model=self.ml_models[0],
prediction_two__ml_model=self.ml_models[1],
) for year in range(*YEAR_RANGE) for round_n in range(ROUND_COUNT)
for match_n in range(MATCH_COUNT)
]
def setUp(self):
today = datetime.now(tz=MELBOURNE_TIMEZONE)
year = today.year
self.match_results_data = fake_match_results_data(
ROW_COUNT, (year, year + 1))
# Save records in DB
ml_model = MLModelFactory(name="tipresias")
for match_data in self.match_results_data.to_dict("records"):
match_date = (match_data["date"].to_pydatetime().replace(
tzinfo=MELBOURNE_TIMEZONE))
match_attrs = {
"start_date_time": match_date,
"round_number": match_data["round_number"],
"venue": match_data["venue"],
}
prediction_attrs = {
"prediction__ml_model":
ml_model,
"prediction__predicted_winner__name":
np.random.choice(
[match_data["home_team"], match_data["away_team"]]),
}
team_match_attrs = {
"home_team_match__team__name": match_data["home_team"],
"away_team_match__team__name": match_data["away_team"],
"home_team_match__score": match_data["home_score"],
"away_team_match__score": match_data["away_score"],
}
FullMatchFactory(**match_attrs, **prediction_attrs,
**team_match_attrs)
self.send_email_command = send_email.Command()
def test_played_without_results(self):
FullMatchFactory(
start_date_time=timezone.localtime() - timedelta(days=1),
home_team_match__score=0,
away_team_match__score=0,
)
FullMatchFactory(
start_date_time=timezone.localtime() - timedelta(days=1),
home_team_match__score=50,
away_team_match__score=80,
)
FullMatchFactory(
start_date_time=timezone.localtime() + timedelta(days=1),
home_team_match__score=0,
away_team_match__score=0,
)
played_matches_without_results = Match.played_without_results()
self.assertEqual(played_matches_without_results.count(), 1)
def main():
"""Set up test DB and seed it with FactoryBoy records."""
assert settings.ENVIRONMENT == "test"
current_year = date.today().year
first_season = current_year - N_SEASONS + 1
seasons_rounds_matches = product(
range(first_season, current_year + 1),
range(1, N_ROUNDS_PER_SEASON + 1),
range(N_MATCHES_PER_ROUND),
)
connections["default"].creation.create_test_db(autoclobber=True)
# create_test_db should set the default DB to test_${DATABASE_NAME}
assert settings.DATABASES["default"]["NAME"] != os.getenv("DATABASE_NAME")
with transaction.atomic():
# We need exactly one principal model, and one of each prediction type
# among competition models.
MLModelFactory(
is_principal=True,
used_in_competitions=True,
prediction_type=PredictionType.MARGIN,
)
MLModelFactory(
used_in_competitions=True, prediction_type=PredictionType.WIN_PROBABILITY
)
for _ in range(N_ML_MODELS - 2):
MLModelFactory()
for season, round_number, _match in seasons_rounds_matches:
# We need some future matches for functionality around predictions,
# so we're arbitrarily making the second half of the current season
# sometime after today
future_match = (
season == date.today().year and round_number > N_ROUNDS_PER_SEASON / 2
)
FullMatchFactory(
with_predictions=True,
year=season,
round_number=round_number,
future=future_match,
)
def test_update_results(self, mock_update_result):
match_results = data_factories.fake_match_results_data()
calls = []
for _idx, match_result in match_results.iterrows():
FullMatchFactory(
home_team_match__score=0,
away_team_match__score=0,
start_date_time=match_result["date"],
round_number=match_result["round_number"],
home_team_match__team__name=match_result["home_team"],
away_team_match__team__name=match_result["away_team"],
venue=match_result["venue"],
)
calls.append(call(match_result))
Match.update_results(match_results)
self.assertEqual(mock_update_result.call_count, len(match_results))
def setUp(self):
self.maxDiff = None
self.client = Client(schema)
ml_models = [MLModelFactory(name=model_name) for model_name in MODEL_NAMES]
self.matches = [
FullMatchFactory(
year=year,
round_number=((idx % 23) + 1),
start_date_time=datetime(
year, 6, (idx % 29) + 1, tzinfo=MELBOURNE_TIMEZONE
),
prediction__ml_model=ml_models[0],
prediction_two__ml_model=ml_models[1],
)
for year in range(*YEAR_RANGE)
for idx in range(ROUND_COUNT)
]
def setUp(self):
self.match_results_data = fake_match_results_data(
seasons=(PREDICTION_YEAR, PREDICTION_YEAR + 1)
)
ml_models = [
MLModel.objects.get(is_principal=True),
MLModel.objects.filter(
is_principal=False, used_in_competitions=True
).first(),
]
for match_data in self.match_results_data.to_dict("records"):
match_date = timezone.localtime(match_data["date"].to_pydatetime())
match_attrs = {
"start_date_time": match_date,
"round_number": match_data["round_number"],
"venue": match_data["venue"],
}
prediction_attrs = {
"prediction__ml_model": ml_models[0],
"prediction__predicted_winner__name": np.random.choice(
[match_data["home_team"], match_data["away_team"]]
),
"prediction_two__ml_model": ml_models[1],
"prediction_two__predicted_winner__name": np.random.choice(
[match_data["home_team"], match_data["away_team"]]
),
}
team_match_attrs = {
"home_team_match__team__name": match_data["home_team"],
"away_team_match__team__name": match_data["away_team"],
"home_team_match__score": match_data["home_score"],
"away_team_match__score": match_data["away_score"],
}
FullMatchFactory(
with_predictions=True,
**match_attrs,
**prediction_attrs,
**team_match_attrs,
)
self.send_email_command = send_email.Command()
def test_update_result(self):
with self.subTest("When the match hasn't been played yet"):
match = FullMatchFactory(
future=True,
with_predictions=True,
home_team_match__score=0,
away_team_match__score=0,
)
match.update_result(pd.DataFrame())
# It doesn't update match scores
score_sum = sum(match.teammatch_set.values_list("score", flat=True))
self.assertEqual(score_sum, 0)
# It doesn't update prediction correctness
self.assertEqual(
match.prediction_set.filter(is_correct__in=[True, False]).count(),
0,
)
# It doesn't update match winner or margin
self.assertIsNone(match.winner)
self.assertIsNone(match.margin)
with self.subTest("When the match doesn't have results yet"):
with self.subTest("and has been played within the last week"):
yesterday = timezone.now() - timedelta(days=1)
match = FullMatchFactory(
with_predictions=True,
start_date_time=yesterday,
home_team_match__score=0,
away_team_match__score=0,
prediction__is_correct=None,
prediction_two__is_correct=None,
)
match.winner = None
match.margin = None
match.update_result(pd.DataFrame())
# It doesn't update match scores
score_sum = sum(match.teammatch_set.values_list("score", flat=True))
self.assertEqual(score_sum, 0)
# It doesn't update prediction correctness
self.assertEqual(
match.prediction_set.filter(is_correct__in=[True, False]).count(),
0,
)
# It doesn't update match winner or margin
self.assertIsNone(match.winner)
self.assertIsNone(match.margin)
with self.subTest("and has been played over a week ago"):
eight_days_ago = timezone.now() - timedelta(days=8)
match = FullMatchFactory(
with_predictions=True,
start_date_time=eight_days_ago,
home_team_match__score=0,
away_team_match__score=0,
prediction__is_correct=None,
prediction_two__is_correct=None,
)
with self.assertRaisesRegex(
AssertionError, "Didn't find any match data rows"
):
match.update_result(pd.DataFrame())
# It doesn't update match scores
score_sum = sum(match.teammatch_set.values_list("score", flat=True))
self.assertEqual(score_sum, 0)
# It doesn't update prediction correctness
self.assertEqual(
match.prediction_set.filter(is_correct__in=[True, False]).count(),
0,
)
match_results = data_factories.fake_match_results_data()
match_result = match_results.iloc[0, :]
match = FullMatchFactory(
with_predictions=True,
home_team_match__score=0,
away_team_match__score=0,
start_date_time=match_result["date"],
round_number=match_result["round_number"],
home_team_match__team__name=match_result["home_team"],
away_team_match__team__name=match_result["away_team"],
)
winner_name = (
match_result["home_team"]
if match_result["home_score"] > match_result["away_score"]
else match_result["away_team"]
)
winner = Team.objects.get(name=winner_name)
match.prediction_set.update(predicted_winner=winner)
# We expect a data frame, so can't reuse the match_result series
match.update_result(match_results.iloc[:1, :])
# It updates match scores
match_scores = set(match.teammatch_set.values_list("score", flat=True))
match_data_scores = set(match_result[["home_score", "away_score"]])
self.assertEqual(match_scores, match_data_scores)
# It updates prediction correctness
self.assertGreaterEqual(match.prediction_set.filter(is_correct=True).count(), 1)
# It updates match winner and margin
winner_is_correct = match.winner == winner
self.assertTrue(winner_is_correct or match.is_draw)
self.assertEqual(match.margin, max(match_scores) - min(match_scores))
def test_fetch_latest_round_stats(self):
ml_models = list(MLModel.objects.all())
YEAR = TWENTY_SEVENTEEN
MONTH = 6
latest_matches = [
FullMatchFactory(
year=YEAR,
round_number=(idx + 1),
start_date_time=datetime(
YEAR, MONTH, (idx % 29) + 1, tzinfo=MELBOURNE_TIMEZONE
),
prediction__ml_model=ml_models[0],
prediction__is_correct=True,
prediction_two__ml_model=ml_models[1],
prediction_two__is_correct=True,
)
for idx in range(ROUND_COUNT)
]
query = """
query QueryType {
fetchLatestRoundStats(mlModelName: "accurate_af") {
seasonYear
roundNumber
modelStats {
modelName
cumulativeCorrectCount
cumulativeMeanAbsoluteError
cumulativeMarginDifference
}
}
}
"""
executed = self.client.execute(query)
data = executed["data"]["fetchLatestRoundStats"]
max_match_round = max([match.round_number for match in latest_matches])
self.assertEqual(data["roundNumber"], max_match_round)
max_match_year = max([match.year for match in latest_matches])
self.assertEqual(max_match_year, data["seasonYear"])
model_stats = data["modelStats"]
self.assertEqual("accurate_af", model_stats["modelName"])
self.assertGreater(model_stats["cumulativeCorrectCount"], 0)
self.assertGreater(model_stats["cumulativeMeanAbsoluteError"], 0)
self.assertGreater(model_stats["cumulativeMarginDifference"], 0)
with self.subTest("when the last matches haven't been played yet"):
DAY = 3
with freeze_time(
f"{YEAR}-0{MONTH}-0{DAY}", tz_offset=-HOURS_FROM_UTC_TO_MELBOURNE
):
past_executed = self.client.execute(query)
data = past_executed["data"]["fetchLatestRoundStats"]
self.assertLess(data["roundNumber"], max_match_round)
# Last played match will be from day before, because "now" and the
# start time for "today's match" are equal
self.assertEqual(data["roundNumber"], DAY - 1)
model_stats = data["modelStats"]
self.assertGreater(model_stats["cumulativeCorrectCount"], 0)
self.assertGreater(model_stats["cumulativeMeanAbsoluteError"], 0)
self.assertGreater(model_stats["cumulativeMarginDifference"], 0)
def test_fetch_latest_round_predictions(self):
ml_models = list(MLModel.objects.all())
year = TWENTY_SEVENTEEN
latest_matches = [
FullMatchFactory(
year=year,
round_number=((idx % 23) + 1),
start_date_time=datetime(
year, 6, (idx % 29) + 1, tzinfo=MELBOURNE_TIMEZONE
),
prediction__ml_model=ml_models[0],
prediction_two__ml_model=ml_models[1],
)
for idx in range(ROUND_COUNT)
]
executed = self.client.execute(
"""
query QueryType {
fetchLatestRoundPredictions {
roundNumber
matches {
startDateTime
predictions { predictedWinner { name } predictedMargin }
winner { name }
homeTeam { name }
awayTeam { name }
}
}
}
"""
)
data = executed["data"]["fetchLatestRoundPredictions"]
max_match_round = max([match.round_number for match in latest_matches])
self.assertEqual(data["roundNumber"], max_match_round)
match_years = [
parser.parse(match["startDateTime"]).year for match in data["matches"]
]
self.assertEqual(np.mean(match_years), year)
with self.subTest("with an mlModelName argument"):
executed_ml_name = self.client.execute(
"""
query QueryType {
fetchLatestRoundPredictions {
roundNumber
matches {
startDateTime
predictions(mlModelName: "accurate_af") {
mlModel { name }
}
}
}
}
"""
)
data = executed_ml_name["data"]["fetchLatestRoundPredictions"]
predictions = itertools.chain.from_iterable(
[match["predictions"] for match in data["matches"]]
)
ml_model_names = [pred["mlModel"]["name"] for pred in predictions]
self.assertEqual(ml_model_names, ["accurate_af"])
def test_fetch_yearly_predictions(self):
year = 2015
ml_model_names = (
Match.objects.filter(start_date_time__year=year)
.distinct("prediction__ml_model__name")
.values_list("prediction__ml_model__name", flat=True)
)
ml_models = list(MLModel.objects.filter(name__in=ml_model_names))
# Have to make sure at least one match has a different round_number to compare
# later rounds to earlier ones
FullMatchFactory(
year=year,
round_number=50,
prediction__ml_model=ml_models[0],
prediction_two__ml_model=ml_models[1],
)
executed = self.client.execute(
"""
query QueryType {
fetchYearlyPredictions(year: 2015) {
seasonYear
predictionModelNames
predictionsByRound {
roundNumber
modelPredictions { modelName, cumulativeCorrectCount }
matches { predictions { isCorrect } }
}
}
}
"""
)
data = executed["data"]["fetchYearlyPredictions"]
self.assertEqual(set(data["predictionModelNames"]), set(ml_model_names))
self.assertEqual(data["seasonYear"], 2015)
predictions = data["predictionsByRound"]
earlier_round = predictions[0]
later_round = predictions[1]
self.assertLessEqual(earlier_round["roundNumber"], later_round["roundNumber"])
earlier_round_cum_counts = [
prediction["cumulativeCorrectCount"]
for prediction in earlier_round["modelPredictions"]
]
earlier_round_correct = [
prediction["isCorrect"]
for match in earlier_round["matches"]
for prediction in match["predictions"]
]
# Regression test to make sure cumulative counts are being calculated correctly
self.assertEqual(sum(earlier_round_cum_counts), sum(earlier_round_correct))
later_round_cum_counts = [
prediction["cumulativeCorrectCount"]
for prediction in later_round["modelPredictions"]
]
later_round_correct = [
prediction["isCorrect"]
for match in later_round["matches"]
for prediction in match["predictions"]
]
# Regression test to make sure cumulative counts are being calculated correctly
self.assertEqual(
sum(earlier_round_correct + later_round_correct),
sum(later_round_cum_counts),
)
self.assertLessEqual(sum(earlier_round_cum_counts), sum(later_round_cum_counts))
def test_fetch_latest_round_metrics(self):
ml_models = list(MLModel.objects.filter(name__in=MODEL_NAMES))
YEAR = TWENTY_SEVENTEEN
MONTH = 6
for round_n in range(ROUND_COUNT):
for match_n in range(MATCH_COUNT):
FullMatchFactory(
with_predictions=True,
year=YEAR,
round_number=(round_n + 1),
start_date_time=timezone.make_aware(
datetime(YEAR, MONTH, (round_n % 29) + 1,
match_n * 5)),
prediction__ml_model=ml_models[0],
prediction__force_correct=True,
prediction_two__ml_model=ml_models[1],
prediction_two__force_correct=True,
)
query = """
query {
fetchLatestRoundMetrics {
season
roundNumber
cumulativeCorrectCount
cumulativeAccuracy
cumulativeMeanAbsoluteError
cumulativeMarginDifference
cumulativeBits
}
}
"""
executed = self.client.execute(query)
data = executed["data"]["fetchLatestRoundMetrics"]
self.assertEqual(data["season"], YEAR)
self.assertEqual(data["roundNumber"], ROUND_COUNT)
# We force all predictions to be correct, so the correct count should just be
# the number of matches
self.assertEqual(data["cumulativeCorrectCount"],
ROUND_COUNT * MATCH_COUNT)
self.assertGreater(data["cumulativeMeanAbsoluteError"], 0)
self.assertEqual(
round(data["cumulativeMarginDifference"]),
round(data["cumulativeMeanAbsoluteError"] * ROUND_COUNT *
MATCH_COUNT),
)
self.assertGreater(data["cumulativeAccuracy"], 0)
# Bits can be positive or negative, so we just want to make sure it's not 0,
# which would suggest a problem
self.assertNotEqual(data["cumulativeBits"], 0)
with self.subTest(
"when the last matches don't have updated results yet"):
TeamMatch.objects.filter(
match__start_date_time__year=YEAR,
match__round_number=ROUND_COUNT).update(score=0)
executed = self.client.execute(query)
data = executed["data"]["fetchLatestRoundMetrics"]
# It fetches latest round with results
self.assertEqual(data["roundNumber"], ROUND_COUNT - 1)
with self.subTest("when the last matches haven't been played yet"):
DAY = 3
fake_datetime = timezone.make_aware(datetime(YEAR, MONTH, DAY))
with freeze_time(fake_datetime):
past_executed = self.client.execute(
query, variables={"mlModelName": "predictanator"})
data = past_executed["data"]["fetchLatestRoundMetrics"]
max_match_round = max(
Match.objects.all(),
key=lambda match: match.round_number).round_number
self.assertLess(data["roundNumber"], max_match_round)
# Last played match will be from day before, because "now" and the
# start time for "today's match" are equal
self.assertEqual(data["roundNumber"], DAY - 1)
self.assertGreater(data["cumulativeCorrectCount"], 0)
self.assertGreater(data["cumulativeMeanAbsoluteError"], 0)
self.assertGreater(data["cumulativeMarginDifference"], 0)
def test_fetch_season_model_metrics_cumulative_metrics(self):
ml_models = list(MLModel.objects.filter(name__in=MODEL_NAMES))
YEAR = TWENTY_SEVENTEEN
MONTH = 6
for round_n in range(ROUND_COUNT):
for match_n in range(MATCH_COUNT):
FullMatchFactory(
with_predictions=True,
year=YEAR,
round_number=(round_n + 1),
start_date_time=timezone.make_aware(
datetime(YEAR, MONTH, (round_n % 29) + 1,
match_n * 5)),
prediction__ml_model=ml_models[0],
prediction__force_correct=True,
prediction_two__ml_model=ml_models[1],
prediction_two__force_correct=True,
)
query = """
query($mlModelName: String) {
fetchSeasonModelMetrics(season: 2017) {
season
roundModelMetrics(roundNumber: -1) {
roundNumber
modelMetrics(mlModelName: $mlModelName) {
mlModel { name }
cumulativeCorrectCount
cumulativeMeanAbsoluteError
cumulativeMarginDifference
cumulativeAccuracy
cumulativeBits
}
}
}
}
"""
with self.subTest("for a 'Win Probability' model"):
executed = self.client.execute(
query, variables={"mlModelName": "accurate_af"})
data = executed["data"]["fetchSeasonModelMetrics"][
"roundModelMetrics"][0]["modelMetrics"]
self.assertEqual(len(data), 1)
model_stats = data[0]
self.assertEqual("accurate_af", model_stats["mlModel"]["name"])
self.assertGreater(model_stats["cumulativeCorrectCount"], 0)
self.assertEqual(model_stats["cumulativeMeanAbsoluteError"], 0)
self.assertEqual(model_stats["cumulativeMarginDifference"], 0)
self.assertGreater(model_stats["cumulativeAccuracy"], 0)
# Bits can be positive or negative, so we just want to make sure it's not 0,
# which would suggest a problem
self.assertNotEqual(model_stats["cumulativeBits"], 0)
with self.subTest("for a 'Margin' model"):
executed = self.client.execute(
query, variables={"mlModelName": "predictanator"})
data = executed["data"]["fetchSeasonModelMetrics"][
"roundModelMetrics"][0]["modelMetrics"]
self.assertEqual(len(data), 1)
model_stats = data[0]
self.assertEqual("predictanator", model_stats["mlModel"]["name"])
self.assertGreater(model_stats["cumulativeCorrectCount"], 0)
self.assertGreater(model_stats["cumulativeMeanAbsoluteError"], 0)
self.assertGreater(model_stats["cumulativeMarginDifference"], 0)
self.assertGreater(model_stats["cumulativeAccuracy"], 0)
self.assertEqual(model_stats["cumulativeBits"], 0)
with self.subTest(
"when the last matches in the DB haven't been played yet"):
DAY = 3
fake_datetime = timezone.make_aware(datetime(YEAR, MONTH, DAY))
with freeze_time(fake_datetime):
# Need to set scores for "future" matches to 0
TeamMatch.objects.filter(
match__start_date_time__gte=fake_datetime).update(score=0)
Match.objects.filter(
start_date_time__gte=fake_datetime).update(winner=None,
margin=None)
past_executed = self.client.execute(
query, variables={"mlModelName": "predictanator"})
data = past_executed["data"]["fetchSeasonModelMetrics"][
"roundModelMetrics"][0]
max_match_round = max(
Match.objects.all(),
key=lambda match: match.round_number).round_number
self.assertLess(data["roundNumber"], max_match_round)
# Last played match will be from day before, because "now" and the
# start time for "today's match" are equal
self.assertEqual(data["roundNumber"], DAY - 1)
model_stats = data["modelMetrics"][0]
self.assertGreater(model_stats["cumulativeCorrectCount"], 0)
self.assertGreater(model_stats["cumulativeMeanAbsoluteError"],
0)
self.assertGreater(model_stats["cumulativeMarginDifference"],
0)
def test_fetch_latest_round_predictions(self):
ml_models = list(MLModel.objects.all())
latest_year = TWENTY_SEVENTEEN
latest_matches = [
FullMatchFactory(
with_predictions=True,
year=latest_year,
round_number=((round_n % 23) + 1),
start_date_time=timezone.make_aware(
datetime(latest_year, 6, (round_n % 29) + 1)),
prediction__ml_model=ml_models[0],
prediction__force_correct=True,
prediction_two__ml_model=ml_models[1],
prediction_two__force_incorrect=True,
) for round_n in range(ROUND_COUNT)
]
query_string = """
query QueryType {
fetchLatestRoundPredictions {
roundNumber
matchPredictions {
startDateTime
predictedWinner
predictedMargin
predictedWinProbability
isCorrect
}
}
}
"""
executed = self.client.execute(query_string)
data = executed["data"]["fetchLatestRoundPredictions"]
# It returns predictions from the last available round
max_match_round = max([match.round_number for match in latest_matches])
self.assertEqual(data["roundNumber"], max_match_round)
# It returns predictions from the last available season
match_years = [
parser.parse(pred["startDateTime"]).year
for pred in data["matchPredictions"]
]
self.assertEqual(np.mean(match_years), latest_year)
# It uses predicted winners from the principal model only
principal_predicted_winners = Prediction.objects.filter(
match__start_date_time__year=latest_year,
match__round_number=max_match_round,
ml_model__is_principal=True,
).values_list("predicted_winner__name", flat=True)
query_predicted_winners = [
pred["predictedWinner"] for pred in data["matchPredictions"]
]
self.assertEqual(sorted(principal_predicted_winners),
sorted(query_predicted_winners))
# When models disagree, it inverts predictions from non-principal models
non_principal_prediction_type = MLModel.objects.get(
is_principal=False, used_in_competitions=True).prediction_type
if non_principal_prediction_type == "Margin":
non_principal_prediction_label = "predictedMargin"
draw_prediction = 0
else:
non_principal_prediction_label = "predictedWinProbability"
draw_prediction = 0.5
predicted_losses = [
pred[non_principal_prediction_label] <= draw_prediction
for pred in data["matchPredictions"]
]
self.assertTrue(all(predicted_losses))
with self.subTest("for unplayed matches"):
max_round_number = max(
[match.round_number for match in latest_matches])
for _ in range(MATCH_COUNT):
FullMatchFactory(
with_predictions=True,
future=True,
round_number=max_round_number + 1,
prediction__ml_model=ml_models[0],
prediction_two__ml_model=ml_models[1],
)
executed = self.client.execute(query_string)
data = executed["data"]["fetchLatestRoundPredictions"]
# It returns isCorrect values of null/None
unique_is_correct_values = {
pred["isCorrect"]
for pred in data["matchPredictions"]
}
self.assertEqual(set([None]), unique_is_correct_values)
with self.subTest("that don't have predictions yet"):
Prediction.objects.filter(
match__start_date_time__gt=timezone.now()).delete()
executed = self.client.execute(query_string)
data = executed["data"]["fetchLatestRoundPredictions"]
# It returns predictions from the last round that has them
self.assertEqual(data["roundNumber"], max_round_number)
with self.subTest("without predictions from a non-principal model"):
Prediction.objects.filter(ml_model__is_principal=False).delete()
executed = self.client.execute(query_string)
data = executed["data"]["fetchLatestRoundPredictions"]
self.assertGreater(len(data["matchPredictions"]), 0)
def test_fetch_season_model_metrics(self):
year = 2015
ml_model_names = (Match.objects.filter(start_date_time__year=year).
distinct("prediction__ml_model__name").values_list(
"prediction__ml_model__name", flat=True))
ml_models = list(MLModel.objects.filter(name__in=ml_model_names))
# Have to make sure at least one match has a different round_number to compare
# later rounds to earlier ones
match_round_number = 50
match_start_date_time = timezone.make_aware(datetime(year, 10, 31))
FullMatchFactory(
with_predictions=True,
year=year,
round_number=match_round_number,
start_date_time=match_start_date_time,
prediction__ml_model=ml_models[0],
prediction_two__ml_model=ml_models[1],
)
query = """
query($season: Int) {
fetchSeasonModelMetrics(season: $season) {
season
roundModelMetrics {
roundNumber
modelMetrics {
mlModel { name }
cumulativeCorrectCount
cumulativeAccuracy
cumulativeMeanAbsoluteError
cumulativeMarginDifference
cumulativeBits
}
}
}
}
"""
variables = {"season": year}
executed = self.client.execute(query, variables=variables)
data = executed["data"]["fetchSeasonModelMetrics"]
self.assertEqual(data["season"], year)
predictions = data["roundModelMetrics"]
for pred in predictions:
for model_metric in pred["modelMetrics"]:
self.assertGreaterEqual(model_metric["cumulativeAccuracy"],
0.0)
self.assertLessEqual(model_metric["cumulativeAccuracy"], 1.0)
earlier_round = predictions[0]
later_round = predictions[1]
self.assertLess(earlier_round["roundNumber"],
later_round["roundNumber"])
earlier_round_cum_correct = [
prediction["cumulativeCorrectCount"]
for prediction in earlier_round["modelMetrics"]
]
earlier_round_cum_accuracy = [
prediction["cumulativeAccuracy"]
for prediction in earlier_round["modelMetrics"]
]
earlier_round_correct = Prediction.objects.filter(
match__start_date_time__year=year,
match__round_number=earlier_round["roundNumber"],
).values_list("is_correct", flat=True)
# Regression tests to make sure cumulative counts and cumulative accuracy
# are being calculated correctly
self.assertEqual(sum(earlier_round_cum_correct),
sum(earlier_round_correct))
self.assertEqual(
round(
sum(earlier_round_cum_accuracy) /
len(earlier_round_cum_accuracy), 4),
round(sum(earlier_round_correct) / len(earlier_round_correct), 4),
)
later_round_cum_correct = [
prediction["cumulativeCorrectCount"]
for prediction in later_round["modelMetrics"]
]
later_round_correct = Prediction.objects.filter(
match__start_date_time__year=year,
match__round_number=later_round["roundNumber"],
).values_list("is_correct", flat=True)
# Regression test to make sure cumulative counts are being calculated correctly
self.assertEqual(
sum(earlier_round_correct) + sum(later_round_correct),
sum(later_round_cum_correct),
)
self.assertLessEqual(sum(earlier_round_cum_correct),
sum(later_round_cum_correct))
with self.subTest("when no matches have been played yet this year"):
unplayed_season = max(YEAR_RANGE)
with freeze_time(
timezone.make_aware(datetime(unplayed_season, 1, 2))):
FullMatchFactory(
with_predictions=True,
year=unplayed_season,
prediction__ml_model=ml_models[0],
prediction_two__ml_model=ml_models[1],
)
executed = self.client.execute(
query, variables={"season": unplayed_season})
self.assertEqual(executed.get("errors"), None)
with self.subTest("with mlModelName argument 'predictanator'"):
executed = self.client.execute("""
query QueryType {
fetchSeasonModelMetrics(season: 2015) {
roundModelMetrics {
modelMetrics(mlModelName: "predictanator") {
mlModel { name }
}
}
}
}
""")
data = executed["data"]["fetchSeasonModelMetrics"][
"roundModelMetrics"][0]
self.assertEqual(len(data["modelMetrics"]), 1)
self.assertEqual(data["modelMetrics"][0]["mlModel"]["name"],
"predictanator")
with self.subTest("with roundNumber argument of -1"):
executed = self.client.execute("""
query QueryType {
fetchSeasonModelMetrics(season: 2015) {
roundModelMetrics(roundNumber: -1) { roundNumber }
}
}
""")
data = executed["data"]["fetchSeasonModelMetrics"][
"roundModelMetrics"]
self.assertEqual(len(data), 1)
max_round_number = max(
Match.objects.all(),
key=lambda match: match.round_number).round_number
self.assertEqual(data[0]["roundNumber"], max_round_number)