def update_scores_1vs1(cls, comparison):
from . import AnswerScore, AnswerCriterionScore, \
ComparisonCriterion, ScoringAlgorithm
assignment_id = comparison.assignment_id
answer1_id = comparison.answer1_id
answer2_id = comparison.answer2_id
# get all other comparisons for the answers not including the ones being calculated
other_comparisons = Comparison.query \
.options(load_only('winner', 'answer1_id', 'answer2_id')) \
.filter(and_(
Comparison.assignment_id == assignment_id,
Comparison.id != comparison.id,
or_(
Comparison.answer1_id.in_([answer1_id, answer2_id]),
Comparison.answer2_id.in_([answer1_id, answer2_id])
)
)) \
.all()
scores = AnswerScore.query \
.filter( AnswerScore.answer_id.in_([answer1_id, answer2_id]) ) \
.all()
# get all other criterion comparisons for the answers not including the ones being calculated
other_criterion_comparisons = ComparisonCriterion.query \
.join("comparison") \
.filter(and_(
Comparison.assignment_id == assignment_id,
~Comparison.id == comparison.id,
or_(
Comparison.answer1_id.in_([answer1_id, answer2_id]),
Comparison.answer2_id.in_([answer1_id, answer2_id])
)
)) \
.all()
criteria_scores = AnswerCriterionScore.query \
.filter( AnswerCriterionScore.answer_id.in_([answer1_id, answer2_id]) ) \
.all()
#update answer criterion scores
updated_criteria_scores = []
for comparison_criterion in comparison.comparison_criteria:
criterion_id = comparison_criterion.criterion_id
score1 = next((criterion_score
for criterion_score in criteria_scores
if criterion_score.answer_id == answer1_id
and criterion_score.criterion_id == criterion_id),
AnswerCriterionScore(assignment_id=assignment_id,
answer_id=answer1_id,
criterion_id=criterion_id))
updated_criteria_scores.append(score1)
key1_scored_object = score1.convert_to_scored_object(
) if score1 != None else ScoredObject(
key=answer1_id,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=0,
opponents=0,
loses=0,
)
score2 = next((criterion_score
for criterion_score in criteria_scores
if criterion_score.answer_id == answer2_id
and criterion_score.criterion_id == criterion_id),
AnswerCriterionScore(assignment_id=assignment_id,
answer_id=answer2_id,
criterion_id=criterion_id))
updated_criteria_scores.append(score2)
key2_scored_object = score2.convert_to_scored_object(
) if score2 != None else ScoredObject(
key=answer2_id,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=0,
opponents=0,
loses=0,
)
result_1, result_2 = calculate_score_1vs1(
package_name=ScoringAlgorithm.elo.value,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=comparison_criterion.comparison_pair_winner(),
other_comparison_pairs=[
c.convert_to_comparison_pair()
for c in other_criterion_comparisons
if c.criterion_id == criterion_id
],
log=current_app.logger)
for score, result in [(score1, result_1), (score2, result_2)]:
score.score = result.score
score.variable1 = result.variable1
score.variable2 = result.variable2
score.rounds = result.rounds
score.wins = result.wins
score.loses = result.loses
score.opponents = result.opponents
updated_scores = []
score1 = next(
(score for score in scores if score.answer_id == answer1_id),
AnswerScore(assignment_id=assignment_id, answer_id=answer1_id))
updated_scores.append(score1)
key1_scored_object = score1.convert_to_scored_object(
) if score1 != None else ScoredObject(
key=answer1_id,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=0,
opponents=0,
loses=0,
)
score2 = next(
(score for score in scores if score.answer_id == answer2_id),
AnswerScore(assignment_id=assignment_id, answer_id=answer2_id))
updated_scores.append(score2)
key2_scored_object = score2.convert_to_scored_object(
) if score2 != None else ScoredObject(
key=answer2_id,
score=None,
variable1=None,
variable2=None,
rounds=0,
wins=0,
opponents=0,
loses=0,
)
result_1, result_2 = calculate_score_1vs1(
package_name=ScoringAlgorithm.elo.value,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=comparison.comparison_pair_winner(),
other_comparison_pairs=[
c.convert_to_comparison_pair() for c in other_comparisons
],
log=current_app.logger)
for score, result in [(score1, result_1), (score2, result_2)]:
score.score = result.score
score.variable1 = result.variable1
score.variable2 = result.variable2
score.rounds = result.rounds
score.wins = result.wins
score.loses = result.loses
score.opponents = result.opponents
db.session.add_all(updated_criteria_scores)
db.session.add_all(updated_scores)
db.session.commit()
return updated_scores
def _run(file_path, pairing_package_name, scoring_package_name, winner_selector, correct_rate):
random.seed()
numpy.random.seed()
actual_grades = numpy.random.normal(0.78, 0.1, NUMBER_OF_ANSWERS)
grade_by_answer_key = {}
answers = []
results = []
for key, grade in enumerate(actual_grades):
grade_by_answer_key[key+1] = grade
answers.append(ScoredObject(
key=key+1, score=0, variable1=None, variable2=None,
rounds=0, opponents=0, wins=0, loses=0
))
students = []
for key in range(NUMBER_OF_STUDENTS):
students.append({
'key': key,
'comparisons_left': NUMBER_OF_COMPARISONS_PER_STUDENT,
'comparisons_completed': []
})
comparisons = []
for round_count in range(1, NUMBER_OF_ROUNDS+1):
if len(students) == 0:
break
for comparison_in_round in range(ROUND_LENGTH):
if len(students) == 0:
break
student = random.choice(students)
student_comparisons = student['comparisons_completed']
comparison_pair = generate_pair(
package_name=pairing_package_name,
scored_objects=answers,
comparison_pairs=student_comparisons
)
key1 = comparison_pair.key1
key1_grade = grade_by_answer_key[key1]
key2 = comparison_pair.key2
key2_grade = grade_by_answer_key[key2]
winner = _decide_winner(
winner_selector, correct_rate,
key1_grade, key2_grade
)
comparison_pair = comparison_pair._replace(winner=winner)
comparisons.append(comparison_pair)
student['comparisons_completed'].append(comparison_pair)
student['comparisons_left'] -= 1
if student['comparisons_left'] <= 0:
indexes = [i for i, s in enumerate(students) if student['key'] == s['key']]
del students[indexes[0]]
index1 = next(index for index, answer in enumerate(answers) if answer.key == key1)
index2 = next(index for index, answer in enumerate(answers) if answer.key == key2)
result1, results2 = calculate_score_1vs1(
package_name=scoring_package_name,
key1_scored_object=answers[index1],
key2_scored_object=answers[index2],
winner=winner,
other_comparison_pairs=comparisons
)
answers[index1] = result1
answers[index2] = results2
current_scores = [answer.score for answer in answers]
r_value, pearsonr_p_value = pearsonr(actual_grades, current_scores)
results.append(str(r_value))
#print("Round {} ----------- pearsonr={} value=={}".format(
# round_count, r_value, pearsonr_p_value
#))
with open(file_path, "a") as csvfile:
out = csv.writer(csvfile)
out.writerow(results)
comparisons.append(comparison_pair)
student['comparisons_completed'].append(comparison_pair)
student[
'comparisons_left'] = student['comparisons_left'] - 1
if student['comparisons_left'] <= 0:
students.remove(student)
index1 = next(index for index, answer in enumerate(answers)
if answer.key == key1)
index2 = next(index for index, answer in enumerate(answers)
if answer.key == key2)
result1, results2 = calculate_score_1vs1(
package_name=scoring_package_name,
key1_scored_object=answers[index1],
key2_scored_object=answers[index2],
winner=winner,
other_comparison_pairs=comparisons)
answers[index1] = result1
answers[index2] = results2
# reverse key order on ties so that we know for sure that the
# score order is eventually correct
answers = sorted(answers,
key=lambda answer:
(answer.score, -answer.key))
completed = True
previous_key = None
top_10_stats = [0, 0, 0, 0, 0]
def _run(file_path, pairing_package_name, scoring_package_name,
winner_selector, correct_rate, actual_grades, repetition_count):
random.seed()
numpy.random.seed()
while repetition_count < REPETITIONS:
grade_by_answer_key = {}
answers = []
results = []
for key, grade in enumerate(actual_grades):
grade_by_answer_key[key + 1] = grade
answers.append(
ScoredObject(key=key + 1,
score=0,
variable1=None,
variable2=None,
rounds=0,
opponents=0,
wins=0,
loses=0))
students = []
for key in range(NUMBER_OF_STUDENTS):
students.append({
'key': key,
'comparisons_left': NUMBER_OF_COMPARISONS_PER_STUDENT,
'comparisons_completed': []
})
comparisons = []
for round_count in range(1, NUMBER_OF_ROUNDS + 1):
if len(students) == 0:
break
for comparison_in_round in range(ROUND_LENGTH):
if len(students) == 0:
break
student = random.choice(students)
student_comparisons = student['comparisons_completed']
comparison_pair = generate_pair(
package_name=pairing_package_name,
scored_objects=answers,
comparison_pairs=student_comparisons)
key1 = comparison_pair.key1
key1_grade = grade_by_answer_key[key1]
key2 = comparison_pair.key2
key2_grade = grade_by_answer_key[key2]
winner = _decide_winner(winner_selector, correct_rate,
key1_grade, key2_grade)
comparison_pair = comparison_pair._replace(winner=winner)
comparisons.append(comparison_pair)
student['comparisons_completed'].append(comparison_pair)
student['comparisons_left'] -= 1
if student['comparisons_left'] <= 0:
indexes = [
i for i, s in enumerate(students)
if student['key'] == s['key']
]
del students[indexes[0]]
index1 = next(index for index, answer in enumerate(answers)
if answer.key == key1)
index2 = next(index for index, answer in enumerate(answers)
if answer.key == key2)
result1, results2 = calculate_score_1vs1(
package_name=scoring_package_name,
key1_scored_object=answers[index1],
key2_scored_object=answers[index2],
winner=winner,
other_comparison_pairs=comparisons)
answers[index1] = result1
answers[index2] = results2
current_scores = [answer.score for answer in answers]
r_value, pearsonr_p_value = pearsonr(ACTUAL_GRADES, current_scores)
results.append(str(r_value))
#print("Round {} ----------- pearsonr={} value=={}".format(
# round_count, r_value, pearsonr_p_value
#))
with open(file_path, "a") as csvfile:
out = csv.writer(csvfile)
out.writerow(results)
# prepare for next run
repetition_count += 1
actual_grades = [answer.score for answer in answers]
def create(assignment_id):
"""Creates report"""
if not assignment_id:
raise RuntimeError(
"Assignment with ID {} is not found.".format(assignment_id))
assignment = Assignment.query.filter_by(id=assignment_id).first()
if not assignment:
raise RuntimeError(
"Assignment with ID {} is not found.".format(assignment_id))
criteria = assignment.criteria
file_name = assignment.course.name.replace('"',
'') + '_' + assignment_id + '_'
answers = Answer.query \
.options(joinedload('score')) \
.options(joinedload('criteria_scores')) \
.filter(and_(
Answer.assignment_id == assignment.id,
Answer.active == True,
Answer.draft == False,
Answer.practice == False
)) \
.order_by(Answer.assignment_id, Answer.user_id) \
.all()
scores = []
for answer in answers:
score = answer.score
if score:
scores.append([
answer.user_id, answer.id, None, 'Overall', score.score,
score.rounds, score.wins, score.loses, score.opponents
])
criteria_scores = answer.criteria_scores
criteria_scores.sort(key=lambda x: x.criterion_id)
for criterion_score in criteria_scores:
criterion = next(criterion for criterion in criteria
if criterion.id == criterion_score.criterion_id)
scores.append([
answer.user_id, answer.id, criterion.id, criterion.name,
criterion_score.score, criterion_score.rounds,
criterion_score.wins, criterion_score.loses,
criterion_score.opponents
])
write_csv(file_name + 'scores_final.csv', [
'User Id', 'Answer Id', 'Criterion Id', 'Criterion', 'Score', 'Rounds',
'Wins', 'Loses', 'Opponents'
], scores)
# replay comparisons for real scores at every step
comparisons_output = []
scores = {criterion.id: {} for criterion in criteria}
scores['overall'] = {}
past_comparisons = {criterion.id: [] for criterion in criteria}
past_comparisons['overall'] = []
comparisons = Comparison.query \
.options(joinedload('comparison_criteria')) \
.filter_by(
assignment_id=assignment.id,
completed=True
) \
.order_by(Comparison.modified, Comparison.user_id) \
.all()
round_length = float(len(answers)) / 2
round_number = 0
for index, comparison in enumerate(comparisons):
answer1_id = comparison.answer1_id
answer2_id = comparison.answer2_id
# overall
answer1_score_before = scores['overall'].get(
answer1_id,
ScoredObject(key=answer1_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
answer2_score_before = scores['overall'].get(
answer2_id,
ScoredObject(key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
other_comparisons = []
for pc in past_comparisons['overall']:
if pc.key1 in [answer1_id, answer2_id
] or pc.key2 in [answer1_id, answer2_id]:
other_comparisons.append(pc)
result_1, result_2 = calculate_score_1vs1(
package_name=ScoringAlgorithm.elo.value,
key1_scored_object=answer1_score_before,
key2_scored_object=answer2_score_before,
winner=comparison.comparison_pair_winner(),
other_comparison_pairs=[c for c in other_comparisons])
scores['overall'][result_1.key] = result_1
scores['overall'][result_2.key] = result_2
answer1_score_after = scores['overall'][answer1_id]
answer2_score_after = scores['overall'][answer2_id]
past_comparisons['overall'].append(
comparison.convert_to_comparison_pair())
winner_id = None
if comparison.winner == WinningAnswer.answer1:
winner_id = answer1_id
elif comparison.winner == WinningAnswer.answer2:
winner_id = answer2_id
elif comparison.winner == WinningAnswer.draw:
winner_id = "draw"
comparisons_output.append([
comparison.user_id, None, 'Overall', answer1_id,
answer1_score_before.score, answer1_score_after.score, answer2_id,
answer2_score_before.score, answer2_score_after.score, winner_id,
comparison.modified
])
# each criterion
comparison_criteria = comparison.comparison_criteria
comparison_criteria.sort(key=lambda x: x.criterion_id)
for comparison_criterion in comparison_criteria:
criterion = next(
criterion for criterion in criteria
if criterion.id == comparison_criterion.criterion_id)
answer1_score_before = scores[criterion.id].get(
answer1_id,
ScoredObject(key=answer1_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
answer2_score_before = scores[criterion.id].get(
answer2_id,
ScoredObject(key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
other_comparisons = []
for pc in past_comparisons[criterion.id]:
if pc.key1 in [answer1_id, answer2_id
] or pc.key2 in [answer1_id, answer2_id]:
other_comparisons.append(pc)
result_1, result_2 = calculate_score_1vs1(
package_name=ScoringAlgorithm.elo.value,
key1_scored_object=answer1_score_before,
key2_scored_object=answer2_score_before,
winner=comparison_criterion.comparison_pair_winner(),
other_comparison_pairs=[c for c in other_comparisons])
scores[criterion.id][result_1.key] = result_1
scores[criterion.id][result_2.key] = result_2
answer1_score_after = scores[criterion.id][answer1_id]
answer2_score_after = scores[criterion.id][answer2_id]
past_comparisons[criterion.id].append(
comparison.convert_to_comparison_pair())
winner_id = None
if comparison_criterion.winner == WinningAnswer.answer1:
winner_id = answer1_id
elif comparison_criterion.winner == WinningAnswer.answer2:
winner_id = answer2_id
comparisons_output.append([
comparison.user_id, criterion.id, criterion.name, answer1_id,
answer1_score_before.score, answer1_score_after.score,
answer2_id, answer2_score_before.score,
answer2_score_after.score, winner_id,
comparison_criterion.modified
])
if (index + 1) % round_length < 1:
round_number += 1
round_scores = []
for answer in answers:
score = scores['overall'].get(
answer.id,
ScoredObject(key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
round_scores.append([
answer.user_id, answer.id, None, 'Overall', score.score,
score.rounds, score.wins, score.loses, score.opponents
])
comparison_criteria = comparison.comparison_criteria
comparison_criteria.sort(key=lambda x: x.criterion_id)
for comparison_criterion in comparison_criteria:
criterion = next(
criterion for criterion in criteria
if criterion.id == comparison_criterion.criterion_id)
criterion_score = scores[criterion.id].get(
answer.id,
ScoredObject(key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0))
round_scores.append([
answer.user_id, answer.id, criterion.id,
criterion.name, criterion_score.score,
criterion_score.rounds, criterion_score.wins,
criterion_score.loses, criterion_score.opponents
])
write_csv(file_name + 'scores_round_' + str(round_number) + '.csv',
[
'User Id', 'Answer Id', 'Criterion Id', 'Criterion',
'Score', 'Rounds', 'Wins', 'Loses', 'Opponents'
], round_scores)
write_csv(file_name + 'comparisons.csv', [
'User Id', 'Criterion Id', 'Criterion', 'Answer 1', 'Score 1 Before',
'Score 1 After', 'Answer 2', 'Score 2 Before', 'Score 2 After',
'Winner', 'Timestamp'
], comparisons_output)
query = User.query \
.join(User.user_courses) \
.with_entities(User.id, User.student_number) \
.filter(UserCourse.course_id == assignment.course_id) \
.order_by(User.id)
users = query.all()
write_csv(file_name + 'users.csv', ['User Id', 'Student #'], users)
print('Done.')
winning_key = select_winner(round, student['key'], key1, key2)
comparison_pair = comparison_pair._replace(winning_key=winning_key)
comparisons.append(comparison_pair)
student['comparisons_completed'].append(comparison_pair)
student['comparisons_left'] = student['comparisons_left'] - 1
if student['comparisons_left'] <= 0:
students.remove(student)
index1 = next(index for index, answer in enumerate(answers) if answer.key == key1)
index2 = next(index for index, answer in enumerate(answers) if answer.key == key2)
result1, results2 = calculate_score_1vs1(
package_name=scoring_package_name,
key1_scored_object=answers[index1],
key2_scored_object=answers[index2],
winning_key=winning_key,
other_comparison_pairs=comparisons
)
answers[index1] = result1
answers[index2] = results2
# reverse key order on ties so that we know for sure that the
# score order is eventually correct
answers = sorted(answers,
key=lambda answer: (
answer.score,
-answer.key
)
)
def test_calculate_score_1vs1(self):
self.comparisons = [
ComparisonPair(1,2,winning_key=1),
ComparisonPair(1,3,winning_key=1)
]
# test comparitive judgement score algorithm
self.package_name = "comparative_judgement"
key1_scored_object = ScoredObject(
key=2, score=None, variable1=None, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
key2_scored_object = ScoredObject(
key=3, score=None, variable1=None, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
winning_key = 2
key2_results, key3_results = calculate_score_1vs1(
package_name=self.package_name,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winning_key=winning_key,
other_comparison_pairs=self.comparisons
)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsNone(key2_results.variable2)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 1)
self.assertIsInstance(key3_results, ScoredObject)
self.assertIsInstance(key3_results.score, float)
self.assertIsInstance(key3_results.variable1, float)
self.assertIsNone(key3_results.variable2)
self.assertEqual(key3_results.rounds, 2)
self.assertEqual(key3_results.opponents, 2)
self.assertEqual(key3_results.wins, 0)
self.assertEqual(key3_results.loses, 2)
# test elo rating algorithm
self.package_name = "elo_rating"
key1_scored_object = ScoredObject(
key=2, score=1395, variable1=1395, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
key2_scored_object = ScoredObject(
key=3, score=1396, variable1=1396, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
winning_key = 2
key2_results, key3_results = calculate_score_1vs1(
package_name=self.package_name,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winning_key=winning_key,
other_comparison_pairs=self.comparisons
)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsNone(key2_results.variable2)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 1)
self.assertIsInstance(key3_results, ScoredObject)
self.assertIsInstance(key3_results.score, float)
self.assertIsInstance(key3_results.variable1, float)
self.assertIsNone(key3_results.variable2)
self.assertEqual(key3_results.rounds, 2)
self.assertEqual(key3_results.opponents, 2)
self.assertEqual(key3_results.wins, 0)
self.assertEqual(key3_results.loses, 2)
# test true skill rating algorithm
self.package_name = "true_skill_rating"
key1_scored_object = ScoredObject(
key=2, score=-0.909, variable1=20.604, variable2=7.171,
rounds=None, wins=None, loses=None, opponents=None
)
key2_scored_object = ScoredObject(
key=3, score=-0.058, variable1=21.542, variable2=7.200,
rounds=None, wins=None, loses=None, opponents=None
)
winning_key = 2
key2_results, key3_results = calculate_score_1vs1(
package_name=self.package_name,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winning_key=winning_key,
other_comparison_pairs=self.comparisons
)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsInstance(key2_results.variable2, float)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 1)
self.assertIsInstance(key3_results, ScoredObject)
self.assertIsInstance(key3_results.score, float)
self.assertIsInstance(key3_results.variable1, float)
self.assertIsInstance(key3_results.variable2, float)
self.assertEqual(key3_results.rounds, 2)
self.assertEqual(key3_results.opponents, 2)
self.assertEqual(key3_results.wins, 0)
self.assertEqual(key3_results.loses, 2)
def create(assignment_id):
"""Creates report"""
if not assignment_id:
raise RuntimeError("Assignment with ID {} is not found.".format(assignment_id))
assignment = Assignment.query.filter_by(id=assignment_id).first()
if not assignment:
raise RuntimeError("Assignment with ID {} is not found.".format(assignment_id))
criteria = assignment.criteria
file_name = assignment.course.name.replace('"', '') + '_' + assignment_id + '_'
answers = Answer.query \
.options(joinedload('score')) \
.options(joinedload('criteria_scores')) \
.filter(and_(
Answer.assignment_id == assignment.id,
Answer.active == True,
Answer.draft == False,
Answer.practice == False
)) \
.order_by(Answer.assignment_id, Answer.user_id) \
.all()
scores = []
for answer in answers:
score = answer.score
if score:
scores.append([answer.user_id, answer.id, None, 'Overall',
score.score, score.rounds, score.wins,
score.loses, score.opponents])
criteria_scores = answer.criteria_scores
criteria_scores.sort(key=lambda x: x.criterion_id)
for criterion_score in criteria_scores:
criterion = next(criterion for criterion in criteria if criterion.id == criterion_score.criterion_id)
scores.append([answer.user_id, answer.id, criterion.id, criterion.name,
criterion_score.score, criterion_score.rounds, criterion_score.wins,
criterion_score.loses, criterion_score.opponents])
write_csv(
file_name + 'scores_final.csv',
['User Id', 'Answer Id', 'Criterion Id', 'Criterion', 'Score', 'Rounds', 'Wins', 'Loses', 'Opponents'],
scores
)
# replay comparisons for real scores at every step
comparisons_output = []
scores = {
criterion.id: {} for criterion in criteria
}
scores['overall'] = {}
past_comparisons = {
criterion.id: [] for criterion in criteria
}
past_comparisons['overall'] = []
comparisons = Comparison.query \
.options(joinedload('comparison_criteria')) \
.filter_by(
assignment_id=assignment.id,
completed=True
) \
.order_by(Comparison.modified, Comparison.user_id) \
.all()
round_length = float(len(answers)) / 2
round_number = 0
for index, comparison in enumerate(comparisons):
answer1_id = comparison.answer1_id
answer2_id = comparison.answer2_id
# overall
answer1_score_before = scores['overall'].get(answer1_id, ScoredObject(
key=answer1_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0
))
answer2_score_before = scores['overall'].get(answer2_id, ScoredObject(
key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0
))
other_comparisons = []
for pc in past_comparisons['overall']:
if pc.key1 in [answer1_id, answer2_id] or pc.key2 in [answer1_id, answer2_id]:
other_comparisons.append(pc)
result_1, result_2 = calculate_score_1vs1(
package_name=ScoringAlgorithm.elo.value,
key1_scored_object=answer1_score_before,
key2_scored_object=answer2_score_before,
winner=comparison.comparison_pair_winner(),
other_comparison_pairs=[c for c in other_comparisons]
)
scores['overall'][result_1.key] = result_1
scores['overall'][result_2.key] = result_2
answer1_score_after = scores['overall'][answer1_id]
answer2_score_after = scores['overall'][answer2_id]
past_comparisons['overall'].append(comparison.convert_to_comparison_pair())
winner_id = None
if comparison.winner == WinningAnswer.answer1:
winner_id = answer1_id
elif comparison.winner == WinningAnswer.answer2:
winner_id = answer2_id
elif comparison.winner == WinningAnswer.draw:
winner_id = "draw"
comparisons_output.append([
comparison.user_id, None, 'Overall',
answer1_id, answer1_score_before.score, answer1_score_after.score,
answer2_id, answer2_score_before.score, answer2_score_after.score,
winner_id, comparison.modified
])
# each criterion
comparison_criteria = comparison.comparison_criteria
comparison_criteria.sort(key=lambda x: x.criterion_id)
for comparison_criterion in comparison_criteria:
criterion = next(criterion for criterion in criteria if criterion.id == comparison_criterion.criterion_id)
answer1_score_before = scores[criterion.id].get(answer1_id, ScoredObject(
key=answer1_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0
))
answer2_score_before = scores[criterion.id].get(answer2_id, ScoredObject(
key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0
))
other_comparisons = []
for pc in past_comparisons[criterion.id]:
if pc.key1 in [answer1_id, answer2_id] or pc.key2 in [answer1_id, answer2_id]:
other_comparisons.append(pc)
result_1, result_2 = calculate_score_1vs1(
package_name=ScoringAlgorithm.elo.value,
key1_scored_object=answer1_score_before,
key2_scored_object=answer2_score_before,
winner=comparison_criterion.comparison_pair_winner(),
other_comparison_pairs=[c for c in other_comparisons]
)
scores[criterion.id][result_1.key] = result_1
scores[criterion.id][result_2.key] = result_2
answer1_score_after = scores[criterion.id][answer1_id]
answer2_score_after = scores[criterion.id][answer2_id]
past_comparisons[criterion.id].append(comparison.convert_to_comparison_pair())
winner_id = None
if comparison_criterion.winner == WinningAnswer.answer1:
winner_id = answer1_id
elif comparison_criterion.winner == WinningAnswer.answer2:
winner_id = answer2_id
comparisons_output.append([
comparison.user_id, criterion.id, criterion.name,
answer1_id, answer1_score_before.score, answer1_score_after.score,
answer2_id, answer2_score_before.score, answer2_score_after.score,
winner_id, comparison_criterion.modified
])
if (index+1) % round_length < 1:
round_number += 1
round_scores = []
for answer in answers:
score = scores['overall'].get(answer.id, ScoredObject(
key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0
))
round_scores.append([answer.user_id, answer.id, None, 'Overall',
score.score, score.rounds, score.wins,
score.loses, score.opponents])
comparison_criteria = comparison.comparison_criteria
comparison_criteria.sort(key=lambda x: x.criterion_id)
for comparison_criterion in comparison_criteria:
criterion = next(criterion for criterion in criteria if criterion.id == comparison_criterion.criterion_id)
criterion_score = scores[criterion.id].get(answer.id, ScoredObject(
key=answer2_id,
score=elo.INITIAL,
variable1=elo.INITIAL,
variable2=None,
rounds=0,
wins=0,
loses=0,
opponents=0
))
round_scores.append([answer.user_id, answer.id, criterion.id, criterion.name,
criterion_score.score, criterion_score.rounds, criterion_score.wins,
criterion_score.loses, criterion_score.opponents])
write_csv(
file_name + 'scores_round_' + str(round_number) + '.csv',
['User Id', 'Answer Id', 'Criterion Id', 'Criterion', 'Score', 'Rounds', 'Wins', 'Loses', 'Opponents'],
round_scores
)
write_csv(
file_name + 'comparisons.csv',
['User Id', 'Criterion Id', 'Criterion',
'Answer 1', 'Score 1 Before', 'Score 1 After',
'Answer 2', 'Score 2 Before', 'Score 2 After',
'Winner', 'Timestamp'],
comparisons_output
)
query = User.query \
.join(User.user_courses) \
.with_entities(User.id, User.student_number) \
.filter(UserCourse.course_id == assignment.course_id) \
.order_by(User.id)
users = query.all()
write_csv(
file_name + 'users.csv',
['User Id', 'Student #'],
users
)
print('Done.')
def update_scores_1vs1(cls, comparison):
from . import AnswerScore, AnswerCriterionScore, \
ComparisonCriterion, ScoringAlgorithm
assignment = comparison.assignment
answer1_id = comparison.answer1_id
answer2_id = comparison.answer2_id
# get all other comparisons for the answers not including the ones being calculated
other_comparisons = Comparison.query \
.options(load_only('winner', 'answer1_id', 'answer2_id')) \
.filter(and_(
Comparison.assignment_id == assignment.id,
Comparison.id != comparison.id,
or_(
Comparison.answer1_id.in_([answer1_id, answer2_id]),
Comparison.answer2_id.in_([answer1_id, answer2_id])
)
)) \
.all()
scores = AnswerScore.query \
.filter( AnswerScore.answer_id.in_([answer1_id, answer2_id]) ) \
.all()
# get all other criterion comparisons for the answers not including the ones being calculated
other_criterion_comparisons = ComparisonCriterion.query \
.join("comparison") \
.filter(and_(
Comparison.assignment_id == assignment.id,
Comparison.id != comparison.id,
or_(
Comparison.answer1_id.in_([answer1_id, answer2_id]),
Comparison.answer2_id.in_([answer1_id, answer2_id])
)
)) \
.all()
criteria_scores = AnswerCriterionScore.query \
.filter(and_(
AnswerCriterionScore.assignment_id == assignment.id,
AnswerCriterionScore.answer_id.in_([answer1_id, answer2_id])
)) \
.all()
#update answer criterion scores
updated_criteria_scores = []
for comparison_criterion in comparison.comparison_criteria:
criterion_id = comparison_criterion.criterion_id
criterion_score1 = next((criterion_score for criterion_score in criteria_scores
if criterion_score.answer_id == answer1_id and criterion_score.criterion_id == criterion_id),
AnswerCriterionScore(assignment_id=assignment.id, answer_id=answer1_id, criterion_id=criterion_id)
)
updated_criteria_scores.append(criterion_score1)
key1_scored_object = criterion_score1.convert_to_scored_object()
criterion_score2 = next((criterion_score for criterion_score in criteria_scores
if criterion_score.answer_id == answer2_id and criterion_score.criterion_id == criterion_id),
AnswerCriterionScore(assignment_id=assignment.id, answer_id=answer2_id, criterion_id=criterion_id)
)
updated_criteria_scores.append(criterion_score2)
key2_scored_object = criterion_score2.convert_to_scored_object()
criterion_result_1, criterion_result_2 = calculate_score_1vs1(
package_name=assignment.scoring_algorithm.value,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=comparison_criterion.comparison_pair_winner(),
other_comparison_pairs=[
cc.convert_to_comparison_pair() for cc in other_criterion_comparisons if cc.criterion_id == criterion_id
],
log=current_app.logger
)
for score, result in [(criterion_score1, criterion_result_1), (criterion_score2, criterion_result_2)]:
score.score = result.score
score.variable1 = result.variable1
score.variable2 = result.variable2
score.rounds = result.rounds
score.wins = result.wins
score.loses = result.loses
score.opponents = result.opponents
updated_scores = []
score1 = next((score for score in scores if score.answer_id == answer1_id),
AnswerScore(assignment_id=assignment.id, answer_id=answer1_id)
)
updated_scores.append(score1)
key1_scored_object = score1.convert_to_scored_object()
score2 = next((score for score in scores if score.answer_id == answer2_id),
AnswerScore(assignment_id=assignment.id, answer_id=answer2_id)
)
updated_scores.append(score2)
key2_scored_object = score2.convert_to_scored_object()
result_1, result_2 = calculate_score_1vs1(
package_name=assignment.scoring_algorithm.value,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=comparison.comparison_pair_winner(),
other_comparison_pairs=[c.convert_to_comparison_pair() for c in other_comparisons],
log=current_app.logger
)
for score, result in [(score1, result_1), (score2, result_2)]:
score.score = result.score
score.variable1 = result.variable1
score.variable2 = result.variable2
score.rounds = result.rounds
score.wins = result.wins
score.loses = result.loses
score.opponents = result.opponents
db.session.add_all(updated_criteria_scores)
db.session.add_all(updated_scores)
db.session.commit()
return updated_scores
def test_calculate_score_1vs1(self):
self.comparisons = [
ComparisonPair(key1=1,key2=2, winner=ComparisonWinner.key1),
ComparisonPair(key1=1,key2=3, winner=ComparisonWinner.key1)
]
# test comparative judgement score algorithm
self.package_name = "comparative_judgement"
key1_scored_object = ScoredObject(
key=2, score=None, variable1=None, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
key2_scored_object = ScoredObject(
key=3, score=None, variable1=None, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
winner = ComparisonWinner.key1
key2_results, key3_results = calculate_score_1vs1(
package_name=self.package_name,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=winner,
other_comparison_pairs=self.comparisons
)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsNone(key2_results.variable2)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 1)
self.assertIsInstance(key3_results, ScoredObject)
self.assertIsInstance(key3_results.score, float)
self.assertIsInstance(key3_results.variable1, float)
self.assertIsNone(key3_results.variable2)
self.assertEqual(key3_results.rounds, 2)
self.assertEqual(key3_results.opponents, 2)
self.assertEqual(key3_results.wins, 0)
self.assertEqual(key3_results.loses, 2)
# test elo rating algorithm
self.package_name = "elo_rating"
key1_scored_object = ScoredObject(
key=2, score=1395, variable1=1395, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
key2_scored_object = ScoredObject(
key=3, score=1396, variable1=1396, variable2=None,
rounds=None, wins=None, loses=None, opponents=None
)
winner = ComparisonWinner.key1
key2_results, key3_results = calculate_score_1vs1(
package_name=self.package_name,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=winner,
other_comparison_pairs=self.comparisons
)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsNone(key2_results.variable2)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 1)
self.assertIsInstance(key3_results, ScoredObject)
self.assertIsInstance(key3_results.score, float)
self.assertIsInstance(key3_results.variable1, float)
self.assertIsNone(key3_results.variable2)
self.assertEqual(key3_results.rounds, 2)
self.assertEqual(key3_results.opponents, 2)
self.assertEqual(key3_results.wins, 0)
self.assertEqual(key3_results.loses, 2)
# test true skill rating algorithm
self.package_name = "true_skill_rating"
key1_scored_object = ScoredObject(
key=2, score=-0.909, variable1=20.604, variable2=7.171,
rounds=None, wins=None, loses=None, opponents=None
)
key2_scored_object = ScoredObject(
key=3, score=-0.058, variable1=21.542, variable2=7.200,
rounds=None, wins=None, loses=None, opponents=None
)
winner = ComparisonWinner.key1
key2_results, key3_results = calculate_score_1vs1(
package_name=self.package_name,
key1_scored_object=key1_scored_object,
key2_scored_object=key2_scored_object,
winner=winner,
other_comparison_pairs=self.comparisons
)
self.assertIsInstance(key2_results, ScoredObject)
self.assertIsInstance(key2_results.score, float)
self.assertIsInstance(key2_results.variable1, float)
self.assertIsInstance(key2_results.variable2, float)
self.assertEqual(key2_results.rounds, 2)
self.assertEqual(key2_results.opponents, 2)
self.assertEqual(key2_results.wins, 1)
self.assertEqual(key2_results.loses, 1)
self.assertIsInstance(key3_results, ScoredObject)
self.assertIsInstance(key3_results.score, float)
self.assertIsInstance(key3_results.variable1, float)
self.assertIsInstance(key3_results.variable2, float)
self.assertEqual(key3_results.rounds, 2)
self.assertEqual(key3_results.opponents, 2)
self.assertEqual(key3_results.wins, 0)
self.assertEqual(key3_results.loses, 2)
