def setUp(self):
self.scored_objects = [
ScoredObject(key=1,
score=0.7,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.2,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None)
]
self.comparisons = []
self.criterion_scores = {
1: {
'c1': 100,
'c2': 200
},
2: {
'c1': 100,
'c2': 200
},
3: {
'c1': 100,
'c2': 200
},
4: {
'c1': 100,
'c2': 200
},
}
self.criterion_weights = {'c1': 0.4, 'c2': 0.6}
def convert_to_scored_object(self):
return ScoredObject(key=self.answer_id,
score=self.score,
variable1=self.variable1,
variable2=self.variable2,
rounds=self.rounds,
wins=self.wins,
loses=self.loses,
opponents=self.opponents)
def setUp(self):
self.scored_objects = [
ScoredObject(key=1,
score=0.7,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.2,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None)
]
self.comparisons = []
def _get_new_comparison_pair(cls, course_id, assignment_id, user_id,
pairing_algorithm, comparisons):
from . import Assignment, UserCourse, CourseRole, Answer, AnswerScore, PairingAlgorithm
# ineligible authors - eg. instructors, TAs, dropped student, current user
non_students = UserCourse.query \
.filter(and_(
UserCourse.course_id == course_id,
UserCourse.course_role != CourseRole.student
))
ineligible_user_ids = [non_student.user_id \
for non_student in non_students]
ineligible_user_ids.append(user_id)
answers_with_score = Answer.query \
.with_entities(Answer, AnswerScore.score ) \
.outerjoin(AnswerScore) \
.filter(and_(
Answer.user_id.notin_(ineligible_user_ids),
Answer.assignment_id == assignment_id,
Answer.active == True,
Answer.practice == False,
Answer.draft == False
)) \
.all()
scored_objects = []
for answer_with_score in answers_with_score:
scored_objects.append(
ScoredObject(key=answer_with_score.Answer.id,
score=answer_with_score.score,
rounds=answer_with_score.Answer.round,
variable1=None,
variable2=None,
wins=None,
loses=None,
opponents=None))
comparison_pairs = [
comparison.convert_to_comparison_pair()
for comparison in comparisons
]
comparison_pair = generate_pair(package_name=pairing_algorithm.value,
scored_objects=scored_objects,
comparison_pairs=comparison_pairs,
log=current_app.logger)
return comparison_pair
def update_scores_1vs1(cls, comparison):
from . import AnswerScore, AnswerCriterionScore, \
ComparisonCriterion, ScoringAlgorithm
assignment_id = comparison.assignment_id
answer1_id = comparison.answer1_id
answer2_id = comparison.answer2_id
# get all other comparisons for the answers not including the ones being calculated
other_comparisons = Comparison.query \
.options(load_only('winner', 'answer1_id', 'answer2_id')) \
.filter(and_(
Comparison.assignment_id == assignment_id,
Comparison.id != comparison.id,
or_(
Comparison.answer1_id.in_([answer1_id, answer2_id]),
Comparison.answer2_id.in_([answer1_id, answer2_id])
)
)) \
.all()
scores = AnswerScore.query \
.filter( AnswerScore.answer_id.in_([answer1_id, answer2_id]) ) \
.all()
# get all other criterion comparisons for the answers not including the ones being calculated
other_criterion_comparisons = ComparisonCriterion.query \
.join("comparison") \
.filter(and_(
Comparison.assignment_id == assignment_id,
~Comparison.id == comparison.id,
or_(
Comparison.answer1_id.in_([answer1_id, answer2_id]),
Comparison.answer2_id.in_([answer1_id, answer2_id])
)
)) \
.all()
criteria_scores = AnswerCriterionScore.query \
.filter( AnswerCriterionScore.answer_id.in_([answer1_id, answer2_id]) ) \
.all()
#update answer criterion scores
updated_criteria_scores = []
for comparison_criterion in comparison.comparison_criteria:
criterion_id = comparison_criterion.criterion_id
score1 = next((criterion_score
for criterion_score in criteria_scores
if criterion_score.answer_id == answer1_id
and criterion_score.criterion_id == criterion_id),
AnswerCriterionScore(assignment_id=assignment_id,
answer_id=answer1_id,
criterion_id=criterion_id))
updated_criteria_scores.append(score1)
key1_scored_object = score1.convert_to_scored_object(
) if score1 != None else ScoredObject(
key=answer1_id,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=0,
opponents=0,
loses=0,
)
score2 = next((criterion_score
for criterion_score in criteria_scores
if criterion_score.answer_id == answer2_id
and criterion_score.criterion_id == criterion_id),
AnswerCriterionScore(assignment_id=assignment_id,
answer_id=answer2_id,
criterion_id=criterion_id))
updated_criteria_scores.append(score2)
key2_scored_object = score2.convert_to_scored_object(
) if score2 != None else ScoredObject(
key=answer2_id,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=0,
opponents=0,
loses=0,
)
result_1, result_2 = calculate_score_1vs1(
package_name=ScoringAlgorithm.elo.value,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=comparison_criterion.comparison_pair_winner(),
other_comparison_pairs=[
c.convert_to_comparison_pair()
for c in other_criterion_comparisons
if c.criterion_id == criterion_id
],
log=current_app.logger)
for score, result in [(score1, result_1), (score2, result_2)]:
score.score = result.score
score.variable1 = result.variable1
score.variable2 = result.variable2
score.rounds = result.rounds
score.wins = result.wins
score.loses = result.loses
score.opponents = result.opponents
updated_scores = []
score1 = next(
(score for score in scores if score.answer_id == answer1_id),
AnswerScore(assignment_id=assignment_id, answer_id=answer1_id))
updated_scores.append(score1)
key1_scored_object = score1.convert_to_scored_object(
) if score1 != None else ScoredObject(
key=answer1_id,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=0,
opponents=0,
loses=0,
)
score2 = next(
(score for score in scores if score.answer_id == answer2_id),
AnswerScore(assignment_id=assignment_id, answer_id=answer2_id))
updated_scores.append(score2)
key2_scored_object = score2.convert_to_scored_object(
) if score2 != None else ScoredObject(
key=answer2_id,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=0,
opponents=0,
loses=0,
)
result_1, result_2 = calculate_score_1vs1(
package_name=ScoringAlgorithm.elo.value,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=comparison.comparison_pair_winner(),
other_comparison_pairs=[
c.convert_to_comparison_pair() for c in other_comparisons
],
log=current_app.logger)
for score, result in [(score1, result_1), (score2, result_2)]:
score.score = result.score
score.variable1 = result.variable1
score.variable2 = result.variable2
score.rounds = result.rounds
score.wins = result.wins
score.loses = result.loses
score.opponents = result.opponents
db.session.add_all(updated_criteria_scores)
db.session.add_all(updated_scores)
db.session.commit()
return updated_scores
# ScoringAlgorithm.comparative_judgement.value,
ScoringAlgorithm.elo.value,
# ScoringAlgorithm.true_skill.value
]
for pairing_package_name in pairing_packages:
for scoring_package_name in scoring_packages:
results = []
for runtime in range(1, TIMES_TO_RUN_PER_ALGORITHM + 1):
answers = []
for key in ANSWER_RANGE:
answers.append(
ScoredObject(key=key,
score=0,
variable1=None,
variable2=None,
rounds=0,
opponents=0,
wins=0,
loses=0))
students = []
for key in STUDENT_RANGE:
students.append({
'key': key,
'comparisons_left': NUMBER_OF_ROUNDS / 2,
'comparisons_completed': []
})
comparisons = []
for round in range(1, NUMBER_OF_ROUNDS + 1):
def test_generate_pair(self, mock_shuffle, mock_random):
##
# empty scored objects set
scored_objects = []
comparisons = []
criterion_scores = {}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
with self.assertRaises(InsufficientObjectsForPairException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
##
# not enough scored objects for comparison (only 1 scored object)
scored_objects = [
ScoredObject(key=1,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
criterion_scores = {1: {'c1': 100, 'c2': 200}}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
with self.assertRaises(InsufficientObjectsForPairException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
##
# User compard all objects error
scored_objects = [
ScoredObject(key=1,
score=0.7,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.2,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None)
]
comparisons = [ComparisonPair(1, 2, None)]
criterion_scores = {
1: {
'c1': 100,
'c2': 200
},
2: {
'c1': 110,
'c2': 220
}
}
criterion_weights = {'c1': 0.7, 'c2': 0.3}
with self.assertRaises(UserComparedAllObjectsException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
##
# User compared all objects error
scored_objects = [
ScoredObject(key=1,
score=0.7,
variable1=None,
variable2=None,
rounds=4,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.2,
variable1=None,
variable2=None,
rounds=4,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.2,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
criterion_scores = {
1: {
'c1': 100,
'c2': 200
},
2: {
'c1': 100,
'c2': 200
},
3: {
'c1': 100,
'c2': 200
},
4: {
'c1': 100,
'c2': 200
}
}
criterion_weights = {'c1': 0.7, 'c2': 0.3}
max_comparisons = 6 # n(n-1)/2 = 4*3/2
for i in range(0, max_comparisons):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
comparisons.append(
ComparisonPair(results.key1, results.key2, results.key1))
# if trying to run one more time, should run into all objects compared error
with self.assertRaises(UserComparedAllObjectsException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
##
# Returns a comparison pair
scored_objects = [
ScoredObject(key=1,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
criterion_scores = {
1: {
'c1': 100,
'c2': 200
},
2: {
'c1': 110,
'c2': 220
}
}
criterion_weights = {'c1': 0.7, 'c2': 0.3}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
self.assertIsInstance(results, ComparisonPair)
min_key = min([results.key1, results.key2])
max_key = max([results.key1, results.key2])
self.assertEqual(min_key, 1)
self.assertEqual(max_key, 2)
self.assertEqual(results.winner, None)
##
# Selects lowest round objects first
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=5,
score=0.5,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=6,
score=0.5,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
criterion_scores = {
1: {
'c1': 100,
'c2': 200
},
2: {
'c1': 100,
'c2': 200
},
3: {
'c1': 100,
'c2': 200
},
4: {
'c1': 100,
'c2': 200
},
5: {
'c1': 100,
'c2': 200
},
6: {
'c1': 100,
'c2': 200
}
}
criterion_weights = {'c1': 0.7, 'c2': 0.3}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
# 5 & 6 should be selected as the lowest valid round objects
min_key = min([results.key1, results.key2])
max_key = max([results.key1, results.key2])
self.assertEqual(min_key, 5)
self.assertEqual(max_key, 6)
##
# Selects lowest criterion score delta
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.9,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
criterion_scores = {
1: {
'c1': 100,
'c2': 100
},
2: {
'c1': 110,
'c2': 110
},
3: {
'c1': 115,
'c2': 100
}
}
criterion_weights = {'c1': 0.7, 'c2': 0.3}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
# 1 should be selected first as it has lowest round.
# it should be paired with 2 as the criterion score delta is lowest.
# (10 * 0.7 + 10 * 0.3 = 10 which is less than 15 * 0.7 = 10.5)
min_key = min([results.key1, results.key2])
max_key = max([results.key1, results.key2])
self.assertEqual(min_key, 1)
self.assertEqual(max_key, 2)
##
# Selects closest score when criterion score deta are the same
scored_objects = [
ScoredObject(key=1,
score=0.9,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.9,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.4,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
criterion_scores = {
1: {
'c1': 11,
'c2': 21
},
2: {
'c1': 11,
'c2': 21
},
3: {
'c1': 10.1,
'c2': 20.2
},
4: {
'c1': 11,
'c2': 21
}
}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
# 3 should be selected first as it has lowest round.
# it should be paired with 4 as the score (0.4) is closer to 0.5.
min_key = min([results.key1, results.key2])
max_key = max([results.key1, results.key2])
self.assertEqual(min_key, 3)
self.assertEqual(max_key, 4)
##
# Selects "randomly" if both criterion scores and scores are the same
scored_objects = [
ScoredObject(key=1,
score=0.9,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.9,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.9,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.9,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
criterion_scores = {
1: {
'c1': 11,
'c2': 21
},
2: {
'c1': 11,
'c2': 21
},
3: {
'c1': 11,
'c2': 21
},
4: {
'c1': 11,
'c2': 21
}
}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
# 4 should be selected first as it has lowest round.
# it should be paired with either 1, 2 or 3.
min_key = min([results.key1, results.key2])
max_key = max([results.key1, results.key2])
# 1 should alwasys be selected because of mock random.random
self.assertEqual(min_key, 1)
self.assertEqual(max_key, 4)
##
# Can select previously compared object but not with same opponent
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = [ComparisonPair(1, 2, None)]
criterion_scores = {
1: {
'c1': 10.1,
'c2': 20.2
},
2: {
'c1': 10.1,
'c2': 20.2
},
3: {
'c1': 10.1,
'c2': 20.2
}
}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
min_key = min([results.key1, results.key2])
max_key = max([results.key1, results.key2])
self.assertEqual(min_key, 1)
self.assertEqual(max_key, 3)
##
# Select opponent with closest score (mock shuffle order)
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.7,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.2,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.4,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
criterion_scores = {
1: {
'c1': 10.1,
'c2': 20.2
},
2: {
'c1': 10.1,
'c2': 20.2
},
3: {
'c1': 10.1,
'c2': 20.2
},
4: {
'c1': 10.1,
'c2': 20.2
}
}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
# object 1 should be selected since random shuffle is disabled
self.assertEqual(results.key1, 1)
# object 4 should be selected as 0.4 is the closest value to 0.5
self.assertEqual(results.key2, 4)
##
# Select opponent with closest score (mock shuffle order)
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.2,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.4,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = [ComparisonPair(1, 2, None), ComparisonPair(3, 4, None)]
criterion_scores = {
1: {
'c1': 10.1,
'c2': 20.2
},
2: {
'c1': 10.1,
'c2': 20.2
},
3: {
'c1': 10.1,
'c2': 20.2
},
4: {
'c1': 10.1,
'c2': 20.2
}
}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
# object 1 should be selected since random shuffle is disabled
self.assertEqual(results.key1, 1)
# object 4 should be selected as 0.4 is the closest value to 0.5 (object 2 already compared, so wont be considered)
self.assertEqual(results.key2, 4)
##
# Select opponent with min delta
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
criterion_scores = {
1: {
'c1': 10.1,
'c2': 20.2
},
2: {
'c1': 10.2,
'c2': 20.3
},
3: {
'c1': 20.1,
'c2': 20.2
},
4: {
'c1': 10.1,
'c2': 30.2
}
}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
# object 1 should be selected since random shuffle is disabled
self.assertEqual(results.key1, 1)
# object 2 should be selected as it has the min weighted criterion score delta
self.assertEqual(results.key2, 2)
##
# Select opponent with closest min delta
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = [ComparisonPair(1, 2, None), ComparisonPair(3, 4, None)]
criterion_scores = {
1: {
'c1': 10.1,
'c2': 20.2
},
2: {
'c1': 10.2,
'c2': 20.3
},
3: {
'c1': 20.1,
'c2': 20.2
},
4: {
'c1': 10.1,
'c2': 30.2
}
}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
# object 1 should be selected since random shuffle is disabled
self.assertEqual(results.key1, 1)
# object 3 should be selected as it has the min weighted criterion scores delta
# after object 2 (which is already compared)
self.assertEqual(results.key2, 3)
##
# Select from long list
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=500,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None)
]
for keyNum in range(2, 500):
scored_objects.append(
ScoredObject(key=keyNum,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None))
comparisons = []
criterion_scores = {
1: {
'c1': 10,
'c2': 20
},
500: {
'c1': 15,
'c2': 24.9
}
}
for keyNum in range(2, 500):
criterion_scores[keyNum] = {'c1': 15, 'c2': 25}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons,
criterion_scores,
criterion_weights)
# object 1 should be selected since it has lowest round
self.assertEqual(results.key1, 1)
# object 500 should be select as it as the lowest criterion score delta
self.assertEqual(results.key2, 500)
# ensure that all scored objects are seen only once until they have almost all been seen
# (even number of scored objects)
scored_objects = [
ScoredObject(key=index,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None) for index in range(30)
]
comparisons = []
criterion_scores = {}
for key in range(30):
criterion_scores[key] = {'c1': None, 'c2': None}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
used_keys = set()
all_keys = set(range(30))
# n/2 comparisons = 15
for _ in range(15):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
# neither key should have been seen before
self.assertNotIn(results.key1, used_keys)
self.assertNotIn(results.key2, used_keys)
comparisons.append(results)
used_keys.add(results.key1)
used_keys.add(results.key2)
self.assertEqual(used_keys, all_keys)
# remaining comparisons for n(n-1)/2 = 435
for _ in range(435 - 15):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
# both keys should have been seen before
self.assertIn(results.key1, used_keys)
self.assertIn(results.key2, used_keys)
comparisons.append(results)
# next comparison should be an UserComparedAllObjectsException error
with self.assertRaises(UserComparedAllObjectsException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
# make sure all pairs are distinct
self.assertEqual(
len(comparisons),
len(set([tuple(sorted([c.key1, c.key2])) for c in comparisons])))
# ensure that all scored objects are seen only once until they have almost all been seen
# (odd number of scored objects)
scored_objects = [
ScoredObject(key=index,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None) for index in range(31)
]
comparisons = []
criterion_scores = {}
for key in range(30):
criterion_scores[key] = {'c1': None, 'c2': None}
criterion_weights = {'c1': 0.4, 'c2': 0.6}
used_keys = set()
all_keys = set(range(31))
# floor(n/2) comparisons = 15
for _ in range(15):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
# neither key should have been seen before
self.assertNotIn(results.key1, used_keys)
self.assertNotIn(results.key2, used_keys)
comparisons.append(results)
used_keys.add(results.key1)
used_keys.add(results.key2)
# there should be only one key missing
self.assertEqual(len(all_keys - used_keys), 1)
# remaining comparisons for n(n-1)/2 = 435
for _ in range(465 - 15):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
# both keys should have been seen before
self.assertIn(results.key1, all_keys)
self.assertIn(results.key2, all_keys)
comparisons.append(results)
# next comparison should be an UserComparedAllObjectsException error
with self.assertRaises(UserComparedAllObjectsException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons, criterion_scores,
criterion_weights)
# make sure all pairs are distinct
self.assertEqual(
len(comparisons),
len(set([tuple(sorted([c.key1, c.key2])) for c in comparisons])))
def create(assignment_id):
"""Creates report"""
if not assignment_id:
raise RuntimeError(
"Assignment with ID {} is not found.".format(assignment_id))
assignment = Assignment.query.filter_by(id=assignment_id).first()
if not assignment:
raise RuntimeError(
"Assignment with ID {} is not found.".format(assignment_id))
criteria = assignment.criteria
file_name = assignment.course.name.replace('"',
'') + '_' + assignment_id + '_'
answers = Answer.query \
.options(joinedload('score')) \
.options(joinedload('criteria_scores')) \
.filter(and_(
Answer.assignment_id == assignment.id,
Answer.active == True,
Answer.draft == False,
Answer.practice == False
)) \
.order_by(Answer.assignment_id, Answer.user_id) \
.all()
scores = []
for answer in answers:
score = answer.score
if score:
scores.append([
answer.user_id, answer.id, None, 'Overall', score.score,
score.rounds, score.wins, score.loses, score.opponents
])
criteria_scores = answer.criteria_scores
criteria_scores.sort(key=lambda x: x.criterion_id)
for criterion_score in criteria_scores:
criterion = next(criterion for criterion in criteria
if criterion.id == criterion_score.criterion_id)
scores.append([
answer.user_id, answer.id, criterion.id, criterion.name,
criterion_score.score, criterion_score.rounds,
criterion_score.wins, criterion_score.loses,
criterion_score.opponents
])
write_csv(file_name + 'scores_final.csv', [
'User Id', 'Answer Id', 'Criterion Id', 'Criterion', 'Score', 'Rounds',
'Wins', 'Loses', 'Opponents'
], scores)
# replay comparisons for real scores at every step
comparisons_output = []
scores = {criterion.id: {} for criterion in criteria}
scores['overall'] = {}
past_comparisons = {criterion.id: [] for criterion in criteria}
past_comparisons['overall'] = []
comparisons = Comparison.query \
.options(joinedload('comparison_criteria')) \
.filter_by(
assignment_id=assignment.id,
completed=True
) \
.order_by(Comparison.modified, Comparison.user_id) \
.all()
round_length = float(len(answers)) / 2
round_number = 0
for index, comparison in enumerate(comparisons):
answer1_id = comparison.answer1_id
answer2_id = comparison.answer2_id
# overall
answer1_score_before = scores['overall'].get(
answer1_id,
ScoredObject(key=answer1_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
answer2_score_before = scores['overall'].get(
answer2_id,
ScoredObject(key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
other_comparisons = []
for pc in past_comparisons['overall']:
if pc.key1 in [answer1_id, answer2_id
] or pc.key2 in [answer1_id, answer2_id]:
other_comparisons.append(pc)
result_1, result_2 = calculate_score_1vs1(
package_name=ScoringAlgorithm.elo.value,
key1_scored_object=answer1_score_before,
key2_scored_object=answer2_score_before,
winner=comparison.comparison_pair_winner(),
other_comparison_pairs=[c for c in other_comparisons])
scores['overall'][result_1.key] = result_1
scores['overall'][result_2.key] = result_2
answer1_score_after = scores['overall'][answer1_id]
answer2_score_after = scores['overall'][answer2_id]
past_comparisons['overall'].append(
comparison.convert_to_comparison_pair())
winner_id = None
if comparison.winner == WinningAnswer.answer1:
winner_id = answer1_id
elif comparison.winner == WinningAnswer.answer2:
winner_id = answer2_id
elif comparison.winner == WinningAnswer.draw:
winner_id = "draw"
comparisons_output.append([
comparison.user_id, None, 'Overall', answer1_id,
answer1_score_before.score, answer1_score_after.score, answer2_id,
answer2_score_before.score, answer2_score_after.score, winner_id,
comparison.modified
])
# each criterion
comparison_criteria = comparison.comparison_criteria
comparison_criteria.sort(key=lambda x: x.criterion_id)
for comparison_criterion in comparison_criteria:
criterion = next(
criterion for criterion in criteria
if criterion.id == comparison_criterion.criterion_id)
answer1_score_before = scores[criterion.id].get(
answer1_id,
ScoredObject(key=answer1_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
answer2_score_before = scores[criterion.id].get(
answer2_id,
ScoredObject(key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
other_comparisons = []
for pc in past_comparisons[criterion.id]:
if pc.key1 in [answer1_id, answer2_id
] or pc.key2 in [answer1_id, answer2_id]:
other_comparisons.append(pc)
result_1, result_2 = calculate_score_1vs1(
package_name=ScoringAlgorithm.elo.value,
key1_scored_object=answer1_score_before,
key2_scored_object=answer2_score_before,
winner=comparison_criterion.comparison_pair_winner(),
other_comparison_pairs=[c for c in other_comparisons])
scores[criterion.id][result_1.key] = result_1
scores[criterion.id][result_2.key] = result_2
answer1_score_after = scores[criterion.id][answer1_id]
answer2_score_after = scores[criterion.id][answer2_id]
past_comparisons[criterion.id].append(
comparison.convert_to_comparison_pair())
winner_id = None
if comparison_criterion.winner == WinningAnswer.answer1:
winner_id = answer1_id
elif comparison_criterion.winner == WinningAnswer.answer2:
winner_id = answer2_id
comparisons_output.append([
comparison.user_id, criterion.id, criterion.name, answer1_id,
answer1_score_before.score, answer1_score_after.score,
answer2_id, answer2_score_before.score,
answer2_score_after.score, winner_id,
comparison_criterion.modified
])
if (index + 1) % round_length < 1:
round_number += 1
round_scores = []
for answer in answers:
score = scores['overall'].get(
answer.id,
ScoredObject(key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
round_scores.append([
answer.user_id, answer.id, None, 'Overall', score.score,
score.rounds, score.wins, score.loses, score.opponents
])
comparison_criteria = comparison.comparison_criteria
comparison_criteria.sort(key=lambda x: x.criterion_id)
for comparison_criterion in comparison_criteria:
criterion = next(
criterion for criterion in criteria
if criterion.id == comparison_criterion.criterion_id)
criterion_score = scores[criterion.id].get(
answer.id,
ScoredObject(key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
round_scores.append([
answer.user_id, answer.id, criterion.id,
criterion.name, criterion_score.score,
criterion_score.rounds, criterion_score.wins,
criterion_score.loses, criterion_score.opponents
])
write_csv(file_name + 'scores_round_' + str(round_number) + '.csv',
[
'User Id', 'Answer Id', 'Criterion Id', 'Criterion',
'Score', 'Rounds', 'Wins', 'Loses', 'Opponents'
], round_scores)
write_csv(file_name + 'comparisons.csv', [
'User Id', 'Criterion Id', 'Criterion', 'Answer 1', 'Score 1 Before',
'Score 1 After', 'Answer 2', 'Score 2 Before', 'Score 2 After',
'Winner', 'Timestamp'
], comparisons_output)
query = User.query \
.join(User.user_courses) \
.with_entities(User.id, User.student_number) \
.filter(UserCourse.course_id == assignment.course_id) \
.order_by(User.id)
users = query.all()
write_csv(file_name + 'users.csv', ['User Id', 'Student #'], users)
print('Done.')
def test_generate_pair(self, mock_shuffle):
# empty scored objects set
scored_objects = []
comparisons = []
with self.assertRaises(InsufficientObjectsForPairException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# not enough scored objects for comparison (only 1 scored object)
scored_objects = [
ScoredObject(key=1,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
with self.assertRaises(InsufficientObjectsForPairException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# User compard all objects error
scored_objects = [
ScoredObject(key=1,
score=0.7,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.2,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None)
]
comparisons = [ComparisonPair(1, 2, None)]
with self.assertRaises(UserComparedAllObjectsException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# User compard all objects error
scored_objects = [
ScoredObject(key=1,
score=0.7,
variable1=None,
variable2=None,
rounds=4,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.2,
variable1=None,
variable2=None,
rounds=4,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.2,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
max_comparisons = 6 # n(n-1)/2 = 4*3/2
for i in range(0, max_comparisons):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
comparisons.append(
ComparisonPair(results.key1, results.key2, results.key1))
# if trying to run one more time, should run into all objects compared error
with self.assertRaises(UserComparedAllObjectsException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# Returns a comparison pair
scored_objects = [
ScoredObject(key=1,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons)
self.assertIsInstance(results, ComparisonPair)
min_key = min([results.key1, results.key2])
max_key = max([results.key1, results.key2])
self.assertEqual(min_key, 1)
self.assertEqual(max_key, 2)
self.assertEqual(results.winner, None)
# Selects lowest round objects first
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=5,
score=0.5,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=6,
score=0.5,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons)
# 5 & 6 should be selected as the lowest valid round objects
min_key = min([results.key1, results.key2])
max_key = max([results.key1, results.key2])
self.assertEqual(min_key, 5)
self.assertEqual(max_key, 6)
# Can select previously compared object but not with same opponent
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = [ComparisonPair(1, 2, None)]
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons)
self.assertEqual(results.key1, 1)
self.assertEqual(results.key2, 3)
# Select opponent with closest score (mock shuffle order)
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.7,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.2,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.4,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons)
# object 1 should be selected since random shuffle is disabled
self.assertEqual(results.key1, 1)
# object 4 should be selected as 0.4 is the closest value to 0.5
self.assertEqual(results.key2, 4)
# Select opponent with closest score (mock shuffle order)
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.2,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.4,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = [ComparisonPair(1, 2, None)]
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons)
# object 1 should be selected since random shuffle is disabled
self.assertEqual(results.key1, 1)
# object 4 should be selected as 0.4 is the closest value to 0.5
self.assertEqual(results.key2, 4)
def _get_new_comparison_pair(cls, course_id, assignment_id, user_id,
group_id, pairing_algorithm, comparisons):
from . import Assignment, UserCourse, CourseRole, Answer, AnswerScore, \
PairingAlgorithm, AnswerCriterionScore, AssignmentCriterion, Group
# exclude current user and those without a proper role.
# note that sys admin (not enrolled in the course and thus no course role) can create answers.
# they are considered eligible
ineligibles = UserCourse.query \
.with_entities(UserCourse.user_id) \
.filter(and_(
UserCourse.course_id == course_id,
UserCourse.course_role == CourseRole.dropped
)) \
.all()
ineligible_user_ids = [
ineligible.user_id for ineligible in ineligibles
]
ineligible_user_ids.append(user_id)
query = Answer.query \
.with_entities(Answer, AnswerScore.score) \
.outerjoin(AnswerScore, AnswerScore.answer_id == Answer.id) \
.filter(and_(
or_(
~Answer.user_id.in_(ineligible_user_ids),
Answer.user_id == None # don't filter out group answers
),
Answer.assignment_id == assignment_id,
Answer.active == True,
Answer.practice == False,
Answer.draft == False,
Answer.comparable == True
))
if group_id:
query = query.filter(Answer.group_id != group_id)
answers_with_score = query.all()
scored_objects = []
for answer_with_score in answers_with_score:
scored_objects.append(
ScoredObject(key=answer_with_score.Answer.id,
score=answer_with_score.score,
rounds=answer_with_score.Answer.round,
variable1=None,
variable2=None,
wins=None,
loses=None,
opponents=None))
comparison_pairs = [
comparison.convert_to_comparison_pair()
for comparison in comparisons
]
# adaptive min delta algo requires extra criterion specific parameters
if pairing_algorithm == PairingAlgorithm.adaptive_min_delta:
# retrieve extra criterion score data
answer_criterion_scores = AnswerCriterionScore.query \
.with_entities(AnswerCriterionScore.answer_id,
AnswerCriterionScore.criterion_id, AnswerCriterionScore.score) \
.join(Answer) \
.filter(and_(
Answer.user_id.notin_(ineligible_user_ids),
Answer.assignment_id == assignment_id,
Answer.active == True,
Answer.practice == False,
Answer.draft == False
)) \
.all()
assignment_criterion_weights = AssignmentCriterion.query \
.with_entities(AssignmentCriterion.criterion_id, AssignmentCriterion.weight) \
.filter(and_(
AssignmentCriterion.assignment_id == assignment_id,
AssignmentCriterion.active == True
)) \
.all()
criterion_scores = {}
for criterion_score in answer_criterion_scores:
scores = criterion_scores.setdefault(criterion_score.answer_id,
{})
scores[criterion_score.criterion_id] = criterion_score.score
criterion_weights = {}
for the_weight in assignment_criterion_weights:
criterion_weights[the_weight.criterion_id] = \
the_weight.weight
comparison_pair = generate_pair(
package_name=pairing_algorithm.value,
scored_objects=scored_objects,
comparison_pairs=comparison_pairs,
criterion_scores=criterion_scores,
criterion_weights=criterion_weights,
log=current_app.logger)
else:
comparison_pair = generate_pair(
package_name=pairing_algorithm.value,
scored_objects=scored_objects,
comparison_pairs=comparison_pairs,
log=current_app.logger)
return comparison_pair
def _run(file_path, pairing_package_name, scoring_package_name,
winner_selector, correct_rate, actual_grades, repetition_count):
random.seed()
numpy.random.seed()
while repetition_count < REPETITIONS:
grade_by_answer_key = {}
answers = []
results = []
for key, grade in enumerate(actual_grades):
grade_by_answer_key[key + 1] = grade
answers.append(
ScoredObject(key=key + 1,
score=0,
variable1=None,
variable2=None,
rounds=0,
opponents=0,
wins=0,
loses=0))
students = []
for key in range(NUMBER_OF_STUDENTS):
students.append({
'key': key,
'comparisons_left': NUMBER_OF_COMPARISONS_PER_STUDENT,
'comparisons_completed': []
})
comparisons = []
for round_count in range(1, NUMBER_OF_ROUNDS + 1):
if len(students) == 0:
break
for comparison_in_round in range(ROUND_LENGTH):
if len(students) == 0:
break
student = random.choice(students)
student_comparisons = student['comparisons_completed']
comparison_pair = generate_pair(
package_name=pairing_package_name,
scored_objects=answers,
comparison_pairs=student_comparisons)
key1 = comparison_pair.key1
key1_grade = grade_by_answer_key[key1]
key2 = comparison_pair.key2
key2_grade = grade_by_answer_key[key2]
winner = _decide_winner(winner_selector, correct_rate,
key1_grade, key2_grade)
comparison_pair = comparison_pair._replace(winner=winner)
comparisons.append(comparison_pair)
student['comparisons_completed'].append(comparison_pair)
student['comparisons_left'] -= 1
if student['comparisons_left'] <= 0:
indexes = [
i for i, s in enumerate(students)
if student['key'] == s['key']
]
del students[indexes[0]]
index1 = next(index for index, answer in enumerate(answers)
if answer.key == key1)
index2 = next(index for index, answer in enumerate(answers)
if answer.key == key2)
result1, results2 = calculate_score_1vs1(
package_name=scoring_package_name,
key1_scored_object=answers[index1],
key2_scored_object=answers[index2],
winner=winner,
other_comparison_pairs=comparisons)
answers[index1] = result1
answers[index2] = results2
current_scores = [answer.score for answer in answers]
r_value, pearsonr_p_value = pearsonr(ACTUAL_GRADES, current_scores)
results.append(str(r_value))
#print("Round {} ----------- pearsonr={} value=={}".format(
# round_count, r_value, pearsonr_p_value
#))
with open(file_path, "a") as csvfile:
out = csv.writer(csvfile)
out.writerow(results)
# prepare for next run
repetition_count += 1
actual_grades = [answer.score for answer in answers]
def test_calculate_score_1vs1(self):
# no winning key error raised
key1_scored_object = ScoredObject(key=1,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = None
comparisons = []
with self.assertRaises(InvalidWinnerException):
self.score_algorithm.calculate_score_1vs1(key1_scored_object,
key2_scored_object,
winner, comparisons)
# empty comparison set
key1_scored_object = ScoredObject(key=1,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.key1
comparisons = []
key1_results, key2_results = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
self.assertIsInstance(key1_results, ScoredObject)
self.assertIsInstance(key1_results.score, float)
self.assertIsInstance(key1_results.variable1, float)
self.assertIsNone(key1_results.variable2)
self.assertEqual(key1_results.rounds, 1)
self.assertEqual(key1_results.opponents, 1)
self.assertEqual(key1_results.wins, 1)
self.assertEqual(key1_results.loses, 0)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsNone(key2_results.variable2)
self.assertEqual(key2_results.rounds, 1)
self.assertEqual(key2_results.opponents, 1)
self.assertEqual(key2_results.wins, 0)
self.assertEqual(key2_results.loses, 1)
self.assertGreater(key1_results.score, key2_results.score)
self.assertGreater(key2_results.score, 0)
self.assertGreater(key1_results.variable1, key2_results.variable1)
self.assertGreater(key2_results.variable1, 0)
# No value given for variables
key1_scored_object = ScoredObject(key=1,
score=None,
variable1=None,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=None,
variable1=None,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.key1
comparisons = []
key1_results, key2_results = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
self.assertEqual(key1_results.rounds, 1)
self.assertEqual(key1_results.opponents, 1)
self.assertEqual(key1_results.wins, 1)
self.assertEqual(key1_results.loses, 0)
self.assertEqual(key2_results.rounds, 1)
self.assertEqual(key2_results.opponents, 1)
self.assertEqual(key2_results.wins, 0)
self.assertEqual(key2_results.loses, 1)
self.assertGreater(key1_results.score, key2_results.score)
self.assertGreater(key2_results.score, 0)
self.assertGreater(key1_results.variable1, key2_results.variable1)
self.assertGreater(key2_results.variable1, 0)
# comparison set without winners
key1_scored_object = ScoredObject(key=1,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.key1
comparisons = [
ComparisonPair(key1=1, key2=2, winner=None),
ComparisonPair(key1=1, key2=2, winner=None),
ComparisonPair(key1=1, key2=2, winner=None)
]
key1_results, key2_results = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
self.assertEqual(key1_results.rounds, 4)
self.assertEqual(key1_results.opponents, 1)
self.assertEqual(key1_results.wins, 1)
self.assertEqual(key1_results.loses, 0)
self.assertEqual(key2_results.rounds, 4)
self.assertEqual(key2_results.opponents, 1)
self.assertEqual(key2_results.wins, 0)
self.assertEqual(key2_results.loses, 1)
self.assertGreater(key1_results.score, key2_results.score)
self.assertGreater(key2_results.score, 0)
self.assertGreater(key1_results.variable1, key2_results.variable1)
self.assertGreater(key2_results.variable1, 0)
# one comparison draw
key1_scored_object = ScoredObject(key=1,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.draw
comparisons = []
key1_results, key2_results = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
self.assertIsInstance(key1_results, ScoredObject)
self.assertIsInstance(key1_results.score, float)
self.assertIsInstance(key1_results.variable1, float)
self.assertIsNone(key1_results.variable2)
self.assertEqual(key1_results.rounds, 1)
self.assertEqual(key1_results.opponents, 1)
self.assertEqual(key1_results.wins, 0)
self.assertEqual(key1_results.loses, 0)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsNone(key2_results.variable2)
self.assertEqual(key2_results.rounds, 1)
self.assertEqual(key2_results.opponents, 1)
self.assertEqual(key2_results.wins, 0)
self.assertEqual(key2_results.loses, 0)
self.assertAlmostEqual(key1_results.score, key2_results.score)
self.assertGreater(key2_results.score, 0)
self.assertAlmostEqual(key1_results.variable1, key2_results.variable1)
self.assertGreater(key2_results.variable1, 0)
# comparison set with scores 1 > 2 ~= 3 > 4
key1_scored_object = ScoredObject(key=1,
score=1405,
variable1=1405,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=1405,
variable1=1405,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.key1
comparisons = [
ComparisonPair(key1=1, key2=3, winner=ComparisonWinner.key1),
ComparisonPair(key1=2, key2=4, winner=ComparisonWinner.key1)
]
key1_results, key2_results = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
self.assertEqual(key1_results.rounds, 2)
self.assertEqual(key1_results.opponents, 2)
self.assertEqual(key1_results.wins, 2)
self.assertEqual(key1_results.loses, 0)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 1)
self.assertGreater(key1_results.score, key2_results.score)
self.assertGreater(key2_results.score, 0)
self.assertGreater(key1_results.variable1, key2_results.variable1)
self.assertGreater(key2_results.variable1, 0)
# comparison set with scores 1 ~= 2 ~= 3 > 4 (with draw)
key1_scored_object = ScoredObject(key=1,
score=1405,
variable1=1405,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=1405,
variable1=1405,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.draw
comparisons = [
ComparisonPair(key1=1, key2=3, winner=ComparisonWinner.key1),
ComparisonPair(key1=2, key2=4, winner=ComparisonWinner.key1)
]
key1_results, key2_results = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
self.assertEqual(key1_results.rounds, 2)
self.assertEqual(key1_results.opponents, 2)
self.assertEqual(key1_results.wins, 1)
self.assertEqual(key1_results.loses, 0)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 0)
self.assertAlmostEqual(key1_results.score, key2_results.score)
self.assertGreater(key2_results.score, 0)
self.assertAlmostEqual(key1_results.variable1, key2_results.variable1)
self.assertGreater(key2_results.variable1, 0)
# multiple comparisons between same pairs
key1_scored_object = ScoredObject(key=1,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.key1
comparisons = []
key1_results_1, key2_results_1 = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
key1_scored_object = ScoredObject(key=1,
score=key1_results_1.score,
variable1=key1_results_1.variable1,
variable2=key1_results_1.variable2,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=key2_results_1.score,
variable1=key2_results_1.variable1,
variable2=key2_results_1.variable2,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.key2
comparisons = [
ComparisonPair(key1=1, key2=2, winner=ComparisonWinner.key1)
]
key1_results_2, key2_results_2 = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
# 1 win should have a higher score than 1 win & 1 lose against same opponent
self.assertGreater(key1_results_1.score, key1_results_2.score)
# 1 lose should have a lower score than 1 win & 1 lose against same opponent
self.assertLess(key2_results_1.score, key2_results_2.score)
# 1 win should have a higher expected score than 1 win & 1 lose against same opponent
self.assertGreater(key1_results_1.variable1, key1_results_2.variable1)
# 1 lose should have a lower expected score than 1 win & 1 lose against same opponent
self.assertLess(key2_results_1.variable1, key2_results_2.variable1)
key1_scored_object = ScoredObject(key=1,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=1400,
variable1=1400,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.key1
comparisons = []
key1_results_1, key2_results_1 = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
key1_scored_object = ScoredObject(key=1,
score=key1_results_1.score,
variable1=key1_results_1.variable1,
variable2=key1_results_1.variable2,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=key2_results_1.score,
variable1=key2_results_1.variable1,
variable2=key2_results_1.variable2,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.key1
comparisons = [
ComparisonPair(key1=1, key2=2, winner=ComparisonWinner.key1)
]
key1_results_2, key2_results_2 = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
# 1 win should have a lower score than 2 wins against same opponent
self.assertLess(key1_results_1.score, key1_results_2.score)
# 1 lose should have a higher score than 2 loses against same opponent
self.assertGreater(key2_results_1.score, key2_results_2.score)
# 1 win should have a lower expected score than 2 wins against same opponent
self.assertLess(key1_results_1.variable1, key1_results_2.variable1)
# 1 lose should have a higher expected score than 2 loses against same opponent
self.assertGreater(key2_results_1.variable1, key2_results_2.variable1)
# adding unrelated comparisons should not effect 1 vs 1 stats results
key1_scored_object = ScoredObject(key=1,
score=1405,
variable1=1405,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
key2_scored_object = ScoredObject(key=2,
score=1405,
variable1=1405,
variable2=None,
rounds=None,
wins=None,
loses=None,
opponents=None)
winner = ComparisonWinner.key1
comparisons = [
ComparisonPair(key1=1, key2=3, winner=ComparisonWinner.key1),
ComparisonPair(key1=2, key2=4, winner=ComparisonWinner.key1)
]
key1_results_1, key2_results_1 = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
comparisons = [
ComparisonPair(key1=3, key2=4, winner=ComparisonWinner.key1),
ComparisonPair(key1=1, key2=3, winner=ComparisonWinner.key1),
ComparisonPair(key1=2, key2=42, winner=ComparisonWinner.key1)
]
key1_results_2, key2_results_2 = self.score_algorithm.calculate_score_1vs1(
key1_scored_object, key2_scored_object, winner, comparisons)
self.assertAlmostEqual(key1_results_1.score, key1_results_2.score)
self.assertAlmostEqual(key1_results_1.variable1,
key1_results_2.variable1)
self.assertAlmostEqual(key1_results_1.variable2,
key1_results_2.variable2)
self.assertEqual(key1_results_1.rounds, key1_results_2.rounds)
self.assertEqual(key1_results_1.opponents, key1_results_2.opponents)
self.assertEqual(key1_results_1.wins, key1_results_2.wins)
self.assertEqual(key1_results_1.loses, key1_results_2.loses)
self.assertAlmostEqual(key2_results_1.score, key2_results_2.score)
self.assertAlmostEqual(key2_results_1.variable1,
key2_results_2.variable1)
self.assertAlmostEqual(key2_results_1.variable2,
key2_results_2.variable2)
self.assertEqual(key2_results_1.rounds, key2_results_2.rounds)
self.assertEqual(key2_results_1.opponents, key2_results_2.opponents)
self.assertEqual(key2_results_1.wins, key2_results_2.wins)
self.assertEqual(key2_results_1.loses, key2_results_2.loses)
def test_generate_pair(self, mock_shuffle):
# empty scored objects set
scored_objects = []
comparisons = []
with self.assertRaises(InsufficientObjectsForPairException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# not enough scored objects for comparison (only 1 scored object)
scored_objects = [
ScoredObject(key=1,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
with self.assertRaises(InsufficientObjectsForPairException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# User compard all objects error
scored_objects = [
ScoredObject(key=1,
score=0.7,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.2,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None)
]
comparisons = [ComparisonPair(1, 2, None)]
with self.assertRaises(UserComparedAllObjectsException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# User compard all objects error
scored_objects = [
ScoredObject(key=1,
score=0.7,
variable1=None,
variable2=None,
rounds=4,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.2,
variable1=None,
variable2=None,
rounds=4,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.2,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
max_comparisons = 6 # n(n-1)/2 = 4*3/2
for i in range(0, max_comparisons):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
comparisons.append(
ComparisonPair(results.key1, results.key2, results.key1))
# if trying to run one more time, should run into all objects compared error
with self.assertRaises(UserComparedAllObjectsException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# Returns a comparison pair
scored_objects = [
ScoredObject(key=1,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons)
self.assertIsInstance(results, ComparisonPair)
min_key = min([results.key1, results.key2])
max_key = max([results.key1, results.key2])
self.assertEqual(min_key, 1)
self.assertEqual(max_key, 2)
self.assertEqual(results.winner, None)
# Selects lowest round objects first
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=5,
score=0.5,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=6,
score=0.5,
variable1=None,
variable2=None,
rounds=1,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons)
# 5 & 6 should be selected as the lowest valid round objects
min_key = min([results.key1, results.key2])
max_key = max([results.key1, results.key2])
self.assertEqual(min_key, 5)
self.assertEqual(max_key, 6)
# Can select previously compared object but not with same opponent
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=2,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = [ComparisonPair(1, 2, None)]
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons)
self.assertEqual(results.key1, 1)
self.assertEqual(results.key2, 3)
# Select opponent with closest score (mock shuffle order)
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.7,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.2,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.4,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = []
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons)
# object 1 should be selected since random shuffle is disabled
self.assertEqual(results.key1, 1)
# object 4 should be selected as 0.4 is the closest value to 0.5
self.assertEqual(results.key2, 4)
# Select opponent with closest score (mock shuffle order)
scored_objects = [
ScoredObject(key=1,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=2,
score=0.5,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=3,
score=0.2,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None),
ScoredObject(key=4,
score=0.4,
variable1=None,
variable2=None,
rounds=3,
wins=None,
loses=None,
opponents=None)
]
comparisons = [ComparisonPair(1, 2, None), ComparisonPair(3, 4, None)]
results = self.pair_algorithm.generate_pair(scored_objects,
comparisons)
# object 1 should be selected since random shuffle is disabled
self.assertEqual(results.key1, 1)
# object 4 should be selected as 0.4 is the closest value to 0.5
self.assertEqual(results.key2, 4)
# ensure that all scored objects are seen only once until they have almost all been seen
# (even number of scored objects)
scored_objects = [
ScoredObject(key=index,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None) for index in range(30)
]
comparisons = []
used_keys = set()
all_keys = set(range(30))
# n/2 comparisons = 15
for _ in range(15):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# neither key should have been seen before
self.assertNotIn(results.key1, used_keys)
self.assertNotIn(results.key2, used_keys)
comparisons.append(results)
used_keys.add(results.key1)
used_keys.add(results.key2)
self.assertEqual(used_keys, all_keys)
# remaining comparisons for n(n-1)/2 = 435
for _ in range(435 - 15):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# both keys should have been seen before
self.assertIn(results.key1, used_keys)
self.assertIn(results.key2, used_keys)
comparisons.append(results)
# next comparison should be an UserComparedAllObjectsException error
with self.assertRaises(UserComparedAllObjectsException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# make sure all pairs are distinct
self.assertEqual(
len(comparisons),
len(set([tuple(sorted([c.key1, c.key2])) for c in comparisons])))
# ensure that all scored objects are seen only once until they have almost all been seen
# (odd number of scored objects)
scored_objects = [
ScoredObject(key=index,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=None,
loses=None,
opponents=None) for index in range(31)
]
comparisons = []
used_keys = set()
all_keys = set(range(31))
# floor(n/2) comparisons = 15
for _ in range(15):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# neither key should have been seen before
self.assertNotIn(results.key1, used_keys)
self.assertNotIn(results.key2, used_keys)
comparisons.append(results)
used_keys.add(results.key1)
used_keys.add(results.key2)
# there should be only one key missing
self.assertEqual(len(all_keys - used_keys), 1)
# remaining comparisons for n(n-1)/2 = 435
for _ in range(465 - 15):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# both keys should have been seen before
self.assertIn(results.key1, all_keys)
self.assertIn(results.key2, all_keys)
comparisons.append(results)
# next comparison should be an UserComparedAllObjectsException error
with self.assertRaises(UserComparedAllObjectsException):
results = self.pair_algorithm.generate_pair(
scored_objects, comparisons)
# make sure all pairs are distinct
self.assertEqual(
len(comparisons),
len(set([tuple(sorted([c.key1, c.key2])) for c in comparisons])))
def test_calculate_score_1vs1(self):
self.comparisons = [
ComparisonPair(key1=1,key2=2, winner=ComparisonWinner.key1),
ComparisonPair(key1=1,key2=3, winner=ComparisonWinner.key1)
]
# test comparative judgement score algorithm
self.package_name = "comparative_judgement"
key1_scored_object = ScoredObject(
key=2, score=None, variable1=None, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
key2_scored_object = ScoredObject(
key=3, score=None, variable1=None, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
winner = ComparisonWinner.key1
key2_results, key3_results = calculate_score_1vs1(
package_name=self.package_name,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=winner,
other_comparison_pairs=self.comparisons
)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsNone(key2_results.variable2)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 1)
self.assertIsInstance(key3_results, ScoredObject)
self.assertIsInstance(key3_results.score, float)
self.assertIsInstance(key3_results.variable1, float)
self.assertIsNone(key3_results.variable2)
self.assertEqual(key3_results.rounds, 2)
self.assertEqual(key3_results.opponents, 2)
self.assertEqual(key3_results.wins, 0)
self.assertEqual(key3_results.loses, 2)
# test elo rating algorithm
self.package_name = "elo_rating"
key1_scored_object = ScoredObject(
key=2, score=1395, variable1=1395, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
key2_scored_object = ScoredObject(
key=3, score=1396, variable1=1396, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
winner = ComparisonWinner.key1
key2_results, key3_results = calculate_score_1vs1(
package_name=self.package_name,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=winner,
other_comparison_pairs=self.comparisons
)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsNone(key2_results.variable2)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 1)
self.assertIsInstance(key3_results, ScoredObject)
self.assertIsInstance(key3_results.score, float)
self.assertIsInstance(key3_results.variable1, float)
self.assertIsNone(key3_results.variable2)
self.assertEqual(key3_results.rounds, 2)
self.assertEqual(key3_results.opponents, 2)
self.assertEqual(key3_results.wins, 0)
self.assertEqual(key3_results.loses, 2)
# test true skill rating algorithm
self.package_name = "true_skill_rating"
key1_scored_object = ScoredObject(
key=2, score=-0.909, variable1=20.604, variable2=7.171,
rounds=None, wins=None, loses=None, opponents=None
)
key2_scored_object = ScoredObject(
key=3, score=-0.058, variable1=21.542, variable2=7.200,
rounds=None, wins=None, loses=None, opponents=None
)
winner = ComparisonWinner.key1
key2_results, key3_results = calculate_score_1vs1(
package_name=self.package_name,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=winner,
other_comparison_pairs=self.comparisons
)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsInstance(key2_results.variable2, float)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 1)
self.assertIsInstance(key3_results, ScoredObject)
self.assertIsInstance(key3_results.score, float)
self.assertIsInstance(key3_results.variable1, float)
self.assertIsInstance(key3_results.variable2, float)
self.assertEqual(key3_results.rounds, 2)
self.assertEqual(key3_results.opponents, 2)
self.assertEqual(key3_results.wins, 0)
self.assertEqual(key3_results.loses, 2)