def update_scores_1vs1(cls, comparison): from . import AnswerScore, AnswerCriterionScore, \ ComparisonCriterion, ScoringAlgorithm assignment_id = comparison.assignment_id answer1_id = comparison.answer1_id answer2_id = comparison.answer2_id # get all other comparisons for the answers not including the ones being calculated other_comparisons = Comparison.query \ .options(load_only('winner', 'answer1_id', 'answer2_id')) \ .filter(and_( Comparison.assignment_id == assignment_id, Comparison.id != comparison.id, or_( Comparison.answer1_id.in_([answer1_id, answer2_id]), Comparison.answer2_id.in_([answer1_id, answer2_id]) ) )) \ .all() scores = AnswerScore.query \ .filter( AnswerScore.answer_id.in_([answer1_id, answer2_id]) ) \ .all() # get all other criterion comparisons for the answers not including the ones being calculated other_criterion_comparisons = ComparisonCriterion.query \ .join("comparison") \ .filter(and_( Comparison.assignment_id == assignment_id, ~Comparison.id == comparison.id, or_( Comparison.answer1_id.in_([answer1_id, answer2_id]), Comparison.answer2_id.in_([answer1_id, answer2_id]) ) )) \ .all() criteria_scores = AnswerCriterionScore.query \ .filter( AnswerCriterionScore.answer_id.in_([answer1_id, answer2_id]) ) \ .all() #update answer criterion scores updated_criteria_scores = [] for comparison_criterion in comparison.comparison_criteria: criterion_id = comparison_criterion.criterion_id score1 = next((criterion_score for criterion_score in criteria_scores if criterion_score.answer_id == answer1_id and criterion_score.criterion_id == criterion_id), AnswerCriterionScore(assignment_id=assignment_id, answer_id=answer1_id, criterion_id=criterion_id)) updated_criteria_scores.append(score1) key1_scored_object = score1.convert_to_scored_object( ) if score1 != None else ScoredObject( key=answer1_id, score=None, variable1=None, variable2=None, rounds=0, wins=0, opponents=0, loses=0, ) score2 = next((criterion_score for criterion_score in criteria_scores if criterion_score.answer_id == answer2_id and criterion_score.criterion_id == criterion_id), AnswerCriterionScore(assignment_id=assignment_id, answer_id=answer2_id, criterion_id=criterion_id)) updated_criteria_scores.append(score2) key2_scored_object = score2.convert_to_scored_object( ) if score2 != None else ScoredObject( key=answer2_id, score=None, variable1=None, variable2=None, rounds=0, wins=0, opponents=0, loses=0, ) result_1, result_2 = calculate_score_1vs1( package_name=ScoringAlgorithm.elo.value, key1_scored_object=key1_scored_object, key2_scored_object=key2_scored_object, winner=comparison_criterion.comparison_pair_winner(), other_comparison_pairs=[ c.convert_to_comparison_pair() for c in other_criterion_comparisons if c.criterion_id == criterion_id ], log=current_app.logger) for score, result in [(score1, result_1), (score2, result_2)]: score.score = result.score score.variable1 = result.variable1 score.variable2 = result.variable2 score.rounds = result.rounds score.wins = result.wins score.loses = result.loses score.opponents = result.opponents updated_scores = [] score1 = next( (score for score in scores if score.answer_id == answer1_id), AnswerScore(assignment_id=assignment_id, answer_id=answer1_id)) updated_scores.append(score1) key1_scored_object = score1.convert_to_scored_object( ) if score1 != None else ScoredObject( key=answer1_id, score=None, variable1=None, variable2=None, rounds=0, wins=0, opponents=0, loses=0, ) score2 = next( (score for score in scores if score.answer_id == answer2_id), AnswerScore(assignment_id=assignment_id, answer_id=answer2_id)) updated_scores.append(score2) key2_scored_object = score2.convert_to_scored_object( ) if score2 != None else ScoredObject( key=answer2_id, score=None, variable1=None, variable2=None, rounds=0, wins=0, opponents=0, loses=0, ) result_1, result_2 = calculate_score_1vs1( package_name=ScoringAlgorithm.elo.value, key1_scored_object=key1_scored_object, key2_scored_object=key2_scored_object, winner=comparison.comparison_pair_winner(), other_comparison_pairs=[ c.convert_to_comparison_pair() for c in other_comparisons ], log=current_app.logger) for score, result in [(score1, result_1), (score2, result_2)]: score.score = result.score score.variable1 = result.variable1 score.variable2 = result.variable2 score.rounds = result.rounds score.wins = result.wins score.loses = result.loses score.opponents = result.opponents db.session.add_all(updated_criteria_scores) db.session.add_all(updated_scores) db.session.commit() return updated_scores
def _run(file_path, pairing_package_name, scoring_package_name, winner_selector, correct_rate): random.seed() numpy.random.seed() actual_grades = numpy.random.normal(0.78, 0.1, NUMBER_OF_ANSWERS) grade_by_answer_key = {} answers = [] results = [] for key, grade in enumerate(actual_grades): grade_by_answer_key[key+1] = grade answers.append(ScoredObject( key=key+1, score=0, variable1=None, variable2=None, rounds=0, opponents=0, wins=0, loses=0 )) students = [] for key in range(NUMBER_OF_STUDENTS): students.append({ 'key': key, 'comparisons_left': NUMBER_OF_COMPARISONS_PER_STUDENT, 'comparisons_completed': [] }) comparisons = [] for round_count in range(1, NUMBER_OF_ROUNDS+1): if len(students) == 0: break for comparison_in_round in range(ROUND_LENGTH): if len(students) == 0: break student = random.choice(students) student_comparisons = student['comparisons_completed'] comparison_pair = generate_pair( package_name=pairing_package_name, scored_objects=answers, comparison_pairs=student_comparisons ) key1 = comparison_pair.key1 key1_grade = grade_by_answer_key[key1] key2 = comparison_pair.key2 key2_grade = grade_by_answer_key[key2] winner = _decide_winner( winner_selector, correct_rate, key1_grade, key2_grade ) comparison_pair = comparison_pair._replace(winner=winner) comparisons.append(comparison_pair) student['comparisons_completed'].append(comparison_pair) student['comparisons_left'] -= 1 if student['comparisons_left'] <= 0: indexes = [i for i, s in enumerate(students) if student['key'] == s['key']] del students[indexes[0]] index1 = next(index for index, answer in enumerate(answers) if answer.key == key1) index2 = next(index for index, answer in enumerate(answers) if answer.key == key2) result1, results2 = calculate_score_1vs1( package_name=scoring_package_name, key1_scored_object=answers[index1], key2_scored_object=answers[index2], winner=winner, other_comparison_pairs=comparisons ) answers[index1] = result1 answers[index2] = results2 current_scores = [answer.score for answer in answers] r_value, pearsonr_p_value = pearsonr(actual_grades, current_scores) results.append(str(r_value)) #print("Round {} ----------- pearsonr={} value=={}".format( # round_count, r_value, pearsonr_p_value #)) with open(file_path, "a") as csvfile: out = csv.writer(csvfile) out.writerow(results)
comparisons.append(comparison_pair) student['comparisons_completed'].append(comparison_pair) student[ 'comparisons_left'] = student['comparisons_left'] - 1 if student['comparisons_left'] <= 0: students.remove(student) index1 = next(index for index, answer in enumerate(answers) if answer.key == key1) index2 = next(index for index, answer in enumerate(answers) if answer.key == key2) result1, results2 = calculate_score_1vs1( package_name=scoring_package_name, key1_scored_object=answers[index1], key2_scored_object=answers[index2], winner=winner, other_comparison_pairs=comparisons) answers[index1] = result1 answers[index2] = results2 # reverse key order on ties so that we know for sure that the # score order is eventually correct answers = sorted(answers, key=lambda answer: (answer.score, -answer.key)) completed = True previous_key = None top_10_stats = [0, 0, 0, 0, 0]
def _run(file_path, pairing_package_name, scoring_package_name, winner_selector, correct_rate, actual_grades, repetition_count): random.seed() numpy.random.seed() while repetition_count < REPETITIONS: grade_by_answer_key = {} answers = [] results = [] for key, grade in enumerate(actual_grades): grade_by_answer_key[key + 1] = grade answers.append( ScoredObject(key=key + 1, score=0, variable1=None, variable2=None, rounds=0, opponents=0, wins=0, loses=0)) students = [] for key in range(NUMBER_OF_STUDENTS): students.append({ 'key': key, 'comparisons_left': NUMBER_OF_COMPARISONS_PER_STUDENT, 'comparisons_completed': [] }) comparisons = [] for round_count in range(1, NUMBER_OF_ROUNDS + 1): if len(students) == 0: break for comparison_in_round in range(ROUND_LENGTH): if len(students) == 0: break student = random.choice(students) student_comparisons = student['comparisons_completed'] comparison_pair = generate_pair( package_name=pairing_package_name, scored_objects=answers, comparison_pairs=student_comparisons) key1 = comparison_pair.key1 key1_grade = grade_by_answer_key[key1] key2 = comparison_pair.key2 key2_grade = grade_by_answer_key[key2] winner = _decide_winner(winner_selector, correct_rate, key1_grade, key2_grade) comparison_pair = comparison_pair._replace(winner=winner) comparisons.append(comparison_pair) student['comparisons_completed'].append(comparison_pair) student['comparisons_left'] -= 1 if student['comparisons_left'] <= 0: indexes = [ i for i, s in enumerate(students) if student['key'] == s['key'] ] del students[indexes[0]] index1 = next(index for index, answer in enumerate(answers) if answer.key == key1) index2 = next(index for index, answer in enumerate(answers) if answer.key == key2) result1, results2 = calculate_score_1vs1( package_name=scoring_package_name, key1_scored_object=answers[index1], key2_scored_object=answers[index2], winner=winner, other_comparison_pairs=comparisons) answers[index1] = result1 answers[index2] = results2 current_scores = [answer.score for answer in answers] r_value, pearsonr_p_value = pearsonr(ACTUAL_GRADES, current_scores) results.append(str(r_value)) #print("Round {} ----------- pearsonr={} value=={}".format( # round_count, r_value, pearsonr_p_value #)) with open(file_path, "a") as csvfile: out = csv.writer(csvfile) out.writerow(results) # prepare for next run repetition_count += 1 actual_grades = [answer.score for answer in answers]
def create(assignment_id): """Creates report""" if not assignment_id: raise RuntimeError( "Assignment with ID {} is not found.".format(assignment_id)) assignment = Assignment.query.filter_by(id=assignment_id).first() if not assignment: raise RuntimeError( "Assignment with ID {} is not found.".format(assignment_id)) criteria = assignment.criteria file_name = assignment.course.name.replace('"', '') + '_' + assignment_id + '_' answers = Answer.query \ .options(joinedload('score')) \ .options(joinedload('criteria_scores')) \ .filter(and_( Answer.assignment_id == assignment.id, Answer.active == True, Answer.draft == False, Answer.practice == False )) \ .order_by(Answer.assignment_id, Answer.user_id) \ .all() scores = [] for answer in answers: score = answer.score if score: scores.append([ answer.user_id, answer.id, None, 'Overall', score.score, score.rounds, score.wins, score.loses, score.opponents ]) criteria_scores = answer.criteria_scores criteria_scores.sort(key=lambda x: x.criterion_id) for criterion_score in criteria_scores: criterion = next(criterion for criterion in criteria if criterion.id == criterion_score.criterion_id) scores.append([ answer.user_id, answer.id, criterion.id, criterion.name, criterion_score.score, criterion_score.rounds, criterion_score.wins, criterion_score.loses, criterion_score.opponents ]) write_csv(file_name + 'scores_final.csv', [ 'User Id', 'Answer Id', 'Criterion Id', 'Criterion', 'Score', 'Rounds', 'Wins', 'Loses', 'Opponents' ], scores) # replay comparisons for real scores at every step comparisons_output = [] scores = {criterion.id: {} for criterion in criteria} scores['overall'] = {} past_comparisons = {criterion.id: [] for criterion in criteria} past_comparisons['overall'] = [] comparisons = Comparison.query \ .options(joinedload('comparison_criteria')) \ .filter_by( assignment_id=assignment.id, completed=True ) \ .order_by(Comparison.modified, Comparison.user_id) \ .all() round_length = float(len(answers)) / 2 round_number = 0 for index, comparison in enumerate(comparisons): answer1_id = comparison.answer1_id answer2_id = comparison.answer2_id # overall answer1_score_before = scores['overall'].get( answer1_id, ScoredObject(key=answer1_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0)) answer2_score_before = scores['overall'].get( answer2_id, ScoredObject(key=answer2_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0)) other_comparisons = [] for pc in past_comparisons['overall']: if pc.key1 in [answer1_id, answer2_id ] or pc.key2 in [answer1_id, answer2_id]: other_comparisons.append(pc) result_1, result_2 = calculate_score_1vs1( package_name=ScoringAlgorithm.elo.value, key1_scored_object=answer1_score_before, key2_scored_object=answer2_score_before, winner=comparison.comparison_pair_winner(), other_comparison_pairs=[c for c in other_comparisons]) scores['overall'][result_1.key] = result_1 scores['overall'][result_2.key] = result_2 answer1_score_after = scores['overall'][answer1_id] answer2_score_after = scores['overall'][answer2_id] past_comparisons['overall'].append( comparison.convert_to_comparison_pair()) winner_id = None if comparison.winner == WinningAnswer.answer1: winner_id = answer1_id elif comparison.winner == WinningAnswer.answer2: winner_id = answer2_id elif comparison.winner == WinningAnswer.draw: winner_id = "draw" comparisons_output.append([ comparison.user_id, None, 'Overall', answer1_id, answer1_score_before.score, answer1_score_after.score, answer2_id, answer2_score_before.score, answer2_score_after.score, winner_id, comparison.modified ]) # each criterion comparison_criteria = comparison.comparison_criteria comparison_criteria.sort(key=lambda x: x.criterion_id) for comparison_criterion in comparison_criteria: criterion = next( criterion for criterion in criteria if criterion.id == comparison_criterion.criterion_id) answer1_score_before = scores[criterion.id].get( answer1_id, ScoredObject(key=answer1_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0)) answer2_score_before = scores[criterion.id].get( answer2_id, ScoredObject(key=answer2_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0)) other_comparisons = [] for pc in past_comparisons[criterion.id]: if pc.key1 in [answer1_id, answer2_id ] or pc.key2 in [answer1_id, answer2_id]: other_comparisons.append(pc) result_1, result_2 = calculate_score_1vs1( package_name=ScoringAlgorithm.elo.value, key1_scored_object=answer1_score_before, key2_scored_object=answer2_score_before, winner=comparison_criterion.comparison_pair_winner(), other_comparison_pairs=[c for c in other_comparisons]) scores[criterion.id][result_1.key] = result_1 scores[criterion.id][result_2.key] = result_2 answer1_score_after = scores[criterion.id][answer1_id] answer2_score_after = scores[criterion.id][answer2_id] past_comparisons[criterion.id].append( comparison.convert_to_comparison_pair()) winner_id = None if comparison_criterion.winner == WinningAnswer.answer1: winner_id = answer1_id elif comparison_criterion.winner == WinningAnswer.answer2: winner_id = answer2_id comparisons_output.append([ comparison.user_id, criterion.id, criterion.name, answer1_id, answer1_score_before.score, answer1_score_after.score, answer2_id, answer2_score_before.score, answer2_score_after.score, winner_id, comparison_criterion.modified ]) if (index + 1) % round_length < 1: round_number += 1 round_scores = [] for answer in answers: score = scores['overall'].get( answer.id, ScoredObject(key=answer2_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0)) round_scores.append([ answer.user_id, answer.id, None, 'Overall', score.score, score.rounds, score.wins, score.loses, score.opponents ]) comparison_criteria = comparison.comparison_criteria comparison_criteria.sort(key=lambda x: x.criterion_id) for comparison_criterion in comparison_criteria: criterion = next( criterion for criterion in criteria if criterion.id == comparison_criterion.criterion_id) criterion_score = scores[criterion.id].get( answer.id, ScoredObject(key=answer2_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0)) round_scores.append([ answer.user_id, answer.id, criterion.id, criterion.name, criterion_score.score, criterion_score.rounds, criterion_score.wins, criterion_score.loses, criterion_score.opponents ]) write_csv(file_name + 'scores_round_' + str(round_number) + '.csv', [ 'User Id', 'Answer Id', 'Criterion Id', 'Criterion', 'Score', 'Rounds', 'Wins', 'Loses', 'Opponents' ], round_scores) write_csv(file_name + 'comparisons.csv', [ 'User Id', 'Criterion Id', 'Criterion', 'Answer 1', 'Score 1 Before', 'Score 1 After', 'Answer 2', 'Score 2 Before', 'Score 2 After', 'Winner', 'Timestamp' ], comparisons_output) query = User.query \ .join(User.user_courses) \ .with_entities(User.id, User.student_number) \ .filter(UserCourse.course_id == assignment.course_id) \ .order_by(User.id) users = query.all() write_csv(file_name + 'users.csv', ['User Id', 'Student #'], users) print('Done.')
winning_key = select_winner(round, student['key'], key1, key2) comparison_pair = comparison_pair._replace(winning_key=winning_key) comparisons.append(comparison_pair) student['comparisons_completed'].append(comparison_pair) student['comparisons_left'] = student['comparisons_left'] - 1 if student['comparisons_left'] <= 0: students.remove(student) index1 = next(index for index, answer in enumerate(answers) if answer.key == key1) index2 = next(index for index, answer in enumerate(answers) if answer.key == key2) result1, results2 = calculate_score_1vs1( package_name=scoring_package_name, key1_scored_object=answers[index1], key2_scored_object=answers[index2], winning_key=winning_key, other_comparison_pairs=comparisons ) answers[index1] = result1 answers[index2] = results2 # reverse key order on ties so that we know for sure that the # score order is eventually correct answers = sorted(answers, key=lambda answer: ( answer.score, -answer.key ) )
def test_calculate_score_1vs1(self): self.comparisons = [ ComparisonPair(1,2,winning_key=1), ComparisonPair(1,3,winning_key=1) ] # test comparitive judgement score algorithm self.package_name = "comparative_judgement" key1_scored_object = ScoredObject( key=2, score=None, variable1=None, variable2=None, rounds=None, wins=None, loses=None, opponents=None ) key2_scored_object = ScoredObject( key=3, score=None, variable1=None, variable2=None, rounds=None, wins=None, loses=None, opponents=None ) winning_key = 2 key2_results, key3_results = calculate_score_1vs1( package_name=self.package_name, key1_scored_object=key1_scored_object, key2_scored_object=key2_scored_object, winning_key=winning_key, other_comparison_pairs=self.comparisons ) self.assertIsInstance(key2_results, ScoredObject) self.assertIsInstance(key2_results.score, float) self.assertIsInstance(key2_results.variable1, float) self.assertIsNone(key2_results.variable2) self.assertEqual(key2_results.rounds, 2) self.assertEqual(key2_results.opponents, 2) self.assertEqual(key2_results.wins, 1) self.assertEqual(key2_results.loses, 1) self.assertIsInstance(key3_results, ScoredObject) self.assertIsInstance(key3_results.score, float) self.assertIsInstance(key3_results.variable1, float) self.assertIsNone(key3_results.variable2) self.assertEqual(key3_results.rounds, 2) self.assertEqual(key3_results.opponents, 2) self.assertEqual(key3_results.wins, 0) self.assertEqual(key3_results.loses, 2) # test elo rating algorithm self.package_name = "elo_rating" key1_scored_object = ScoredObject( key=2, score=1395, variable1=1395, variable2=None, rounds=None, wins=None, loses=None, opponents=None ) key2_scored_object = ScoredObject( key=3, score=1396, variable1=1396, variable2=None, rounds=None, wins=None, loses=None, opponents=None ) winning_key = 2 key2_results, key3_results = calculate_score_1vs1( package_name=self.package_name, key1_scored_object=key1_scored_object, key2_scored_object=key2_scored_object, winning_key=winning_key, other_comparison_pairs=self.comparisons ) self.assertIsInstance(key2_results, ScoredObject) self.assertIsInstance(key2_results.score, float) self.assertIsInstance(key2_results.variable1, float) self.assertIsNone(key2_results.variable2) self.assertEqual(key2_results.rounds, 2) self.assertEqual(key2_results.opponents, 2) self.assertEqual(key2_results.wins, 1) self.assertEqual(key2_results.loses, 1) self.assertIsInstance(key3_results, ScoredObject) self.assertIsInstance(key3_results.score, float) self.assertIsInstance(key3_results.variable1, float) self.assertIsNone(key3_results.variable2) self.assertEqual(key3_results.rounds, 2) self.assertEqual(key3_results.opponents, 2) self.assertEqual(key3_results.wins, 0) self.assertEqual(key3_results.loses, 2) # test true skill rating algorithm self.package_name = "true_skill_rating" key1_scored_object = ScoredObject( key=2, score=-0.909, variable1=20.604, variable2=7.171, rounds=None, wins=None, loses=None, opponents=None ) key2_scored_object = ScoredObject( key=3, score=-0.058, variable1=21.542, variable2=7.200, rounds=None, wins=None, loses=None, opponents=None ) winning_key = 2 key2_results, key3_results = calculate_score_1vs1( package_name=self.package_name, key1_scored_object=key1_scored_object, key2_scored_object=key2_scored_object, winning_key=winning_key, other_comparison_pairs=self.comparisons ) self.assertIsInstance(key2_results, ScoredObject) self.assertIsInstance(key2_results.score, float) self.assertIsInstance(key2_results.variable1, float) self.assertIsInstance(key2_results.variable2, float) self.assertEqual(key2_results.rounds, 2) self.assertEqual(key2_results.opponents, 2) self.assertEqual(key2_results.wins, 1) self.assertEqual(key2_results.loses, 1) self.assertIsInstance(key3_results, ScoredObject) self.assertIsInstance(key3_results.score, float) self.assertIsInstance(key3_results.variable1, float) self.assertIsInstance(key3_results.variable2, float) self.assertEqual(key3_results.rounds, 2) self.assertEqual(key3_results.opponents, 2) self.assertEqual(key3_results.wins, 0) self.assertEqual(key3_results.loses, 2)
def create(assignment_id): """Creates report""" if not assignment_id: raise RuntimeError("Assignment with ID {} is not found.".format(assignment_id)) assignment = Assignment.query.filter_by(id=assignment_id).first() if not assignment: raise RuntimeError("Assignment with ID {} is not found.".format(assignment_id)) criteria = assignment.criteria file_name = assignment.course.name.replace('"', '') + '_' + assignment_id + '_' answers = Answer.query \ .options(joinedload('score')) \ .options(joinedload('criteria_scores')) \ .filter(and_( Answer.assignment_id == assignment.id, Answer.active == True, Answer.draft == False, Answer.practice == False )) \ .order_by(Answer.assignment_id, Answer.user_id) \ .all() scores = [] for answer in answers: score = answer.score if score: scores.append([answer.user_id, answer.id, None, 'Overall', score.score, score.rounds, score.wins, score.loses, score.opponents]) criteria_scores = answer.criteria_scores criteria_scores.sort(key=lambda x: x.criterion_id) for criterion_score in criteria_scores: criterion = next(criterion for criterion in criteria if criterion.id == criterion_score.criterion_id) scores.append([answer.user_id, answer.id, criterion.id, criterion.name, criterion_score.score, criterion_score.rounds, criterion_score.wins, criterion_score.loses, criterion_score.opponents]) write_csv( file_name + 'scores_final.csv', ['User Id', 'Answer Id', 'Criterion Id', 'Criterion', 'Score', 'Rounds', 'Wins', 'Loses', 'Opponents'], scores ) # replay comparisons for real scores at every step comparisons_output = [] scores = { criterion.id: {} for criterion in criteria } scores['overall'] = {} past_comparisons = { criterion.id: [] for criterion in criteria } past_comparisons['overall'] = [] comparisons = Comparison.query \ .options(joinedload('comparison_criteria')) \ .filter_by( assignment_id=assignment.id, completed=True ) \ .order_by(Comparison.modified, Comparison.user_id) \ .all() round_length = float(len(answers)) / 2 round_number = 0 for index, comparison in enumerate(comparisons): answer1_id = comparison.answer1_id answer2_id = comparison.answer2_id # overall answer1_score_before = scores['overall'].get(answer1_id, ScoredObject( key=answer1_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0 )) answer2_score_before = scores['overall'].get(answer2_id, ScoredObject( key=answer2_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0 )) other_comparisons = [] for pc in past_comparisons['overall']: if pc.key1 in [answer1_id, answer2_id] or pc.key2 in [answer1_id, answer2_id]: other_comparisons.append(pc) result_1, result_2 = calculate_score_1vs1( package_name=ScoringAlgorithm.elo.value, key1_scored_object=answer1_score_before, key2_scored_object=answer2_score_before, winner=comparison.comparison_pair_winner(), other_comparison_pairs=[c for c in other_comparisons] ) scores['overall'][result_1.key] = result_1 scores['overall'][result_2.key] = result_2 answer1_score_after = scores['overall'][answer1_id] answer2_score_after = scores['overall'][answer2_id] past_comparisons['overall'].append(comparison.convert_to_comparison_pair()) winner_id = None if comparison.winner == WinningAnswer.answer1: winner_id = answer1_id elif comparison.winner == WinningAnswer.answer2: winner_id = answer2_id elif comparison.winner == WinningAnswer.draw: winner_id = "draw" comparisons_output.append([ comparison.user_id, None, 'Overall', answer1_id, answer1_score_before.score, answer1_score_after.score, answer2_id, answer2_score_before.score, answer2_score_after.score, winner_id, comparison.modified ]) # each criterion comparison_criteria = comparison.comparison_criteria comparison_criteria.sort(key=lambda x: x.criterion_id) for comparison_criterion in comparison_criteria: criterion = next(criterion for criterion in criteria if criterion.id == comparison_criterion.criterion_id) answer1_score_before = scores[criterion.id].get(answer1_id, ScoredObject( key=answer1_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0 )) answer2_score_before = scores[criterion.id].get(answer2_id, ScoredObject( key=answer2_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0 )) other_comparisons = [] for pc in past_comparisons[criterion.id]: if pc.key1 in [answer1_id, answer2_id] or pc.key2 in [answer1_id, answer2_id]: other_comparisons.append(pc) result_1, result_2 = calculate_score_1vs1( package_name=ScoringAlgorithm.elo.value, key1_scored_object=answer1_score_before, key2_scored_object=answer2_score_before, winner=comparison_criterion.comparison_pair_winner(), other_comparison_pairs=[c for c in other_comparisons] ) scores[criterion.id][result_1.key] = result_1 scores[criterion.id][result_2.key] = result_2 answer1_score_after = scores[criterion.id][answer1_id] answer2_score_after = scores[criterion.id][answer2_id] past_comparisons[criterion.id].append(comparison.convert_to_comparison_pair()) winner_id = None if comparison_criterion.winner == WinningAnswer.answer1: winner_id = answer1_id elif comparison_criterion.winner == WinningAnswer.answer2: winner_id = answer2_id comparisons_output.append([ comparison.user_id, criterion.id, criterion.name, answer1_id, answer1_score_before.score, answer1_score_after.score, answer2_id, answer2_score_before.score, answer2_score_after.score, winner_id, comparison_criterion.modified ]) if (index+1) % round_length < 1: round_number += 1 round_scores = [] for answer in answers: score = scores['overall'].get(answer.id, ScoredObject( key=answer2_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0 )) round_scores.append([answer.user_id, answer.id, None, 'Overall', score.score, score.rounds, score.wins, score.loses, score.opponents]) comparison_criteria = comparison.comparison_criteria comparison_criteria.sort(key=lambda x: x.criterion_id) for comparison_criterion in comparison_criteria: criterion = next(criterion for criterion in criteria if criterion.id == comparison_criterion.criterion_id) criterion_score = scores[criterion.id].get(answer.id, ScoredObject( key=answer2_id, score=elo.INITIAL, variable1=elo.INITIAL, variable2=None, rounds=0, wins=0, loses=0, opponents=0 )) round_scores.append([answer.user_id, answer.id, criterion.id, criterion.name, criterion_score.score, criterion_score.rounds, criterion_score.wins, criterion_score.loses, criterion_score.opponents]) write_csv( file_name + 'scores_round_' + str(round_number) + '.csv', ['User Id', 'Answer Id', 'Criterion Id', 'Criterion', 'Score', 'Rounds', 'Wins', 'Loses', 'Opponents'], round_scores ) write_csv( file_name + 'comparisons.csv', ['User Id', 'Criterion Id', 'Criterion', 'Answer 1', 'Score 1 Before', 'Score 1 After', 'Answer 2', 'Score 2 Before', 'Score 2 After', 'Winner', 'Timestamp'], comparisons_output ) query = User.query \ .join(User.user_courses) \ .with_entities(User.id, User.student_number) \ .filter(UserCourse.course_id == assignment.course_id) \ .order_by(User.id) users = query.all() write_csv( file_name + 'users.csv', ['User Id', 'Student #'], users ) print('Done.')
def update_scores_1vs1(cls, comparison): from . import AnswerScore, AnswerCriterionScore, \ ComparisonCriterion, ScoringAlgorithm assignment = comparison.assignment answer1_id = comparison.answer1_id answer2_id = comparison.answer2_id # get all other comparisons for the answers not including the ones being calculated other_comparisons = Comparison.query \ .options(load_only('winner', 'answer1_id', 'answer2_id')) \ .filter(and_( Comparison.assignment_id == assignment.id, Comparison.id != comparison.id, or_( Comparison.answer1_id.in_([answer1_id, answer2_id]), Comparison.answer2_id.in_([answer1_id, answer2_id]) ) )) \ .all() scores = AnswerScore.query \ .filter( AnswerScore.answer_id.in_([answer1_id, answer2_id]) ) \ .all() # get all other criterion comparisons for the answers not including the ones being calculated other_criterion_comparisons = ComparisonCriterion.query \ .join("comparison") \ .filter(and_( Comparison.assignment_id == assignment.id, Comparison.id != comparison.id, or_( Comparison.answer1_id.in_([answer1_id, answer2_id]), Comparison.answer2_id.in_([answer1_id, answer2_id]) ) )) \ .all() criteria_scores = AnswerCriterionScore.query \ .filter(and_( AnswerCriterionScore.assignment_id == assignment.id, AnswerCriterionScore.answer_id.in_([answer1_id, answer2_id]) )) \ .all() #update answer criterion scores updated_criteria_scores = [] for comparison_criterion in comparison.comparison_criteria: criterion_id = comparison_criterion.criterion_id criterion_score1 = next((criterion_score for criterion_score in criteria_scores if criterion_score.answer_id == answer1_id and criterion_score.criterion_id == criterion_id), AnswerCriterionScore(assignment_id=assignment.id, answer_id=answer1_id, criterion_id=criterion_id) ) updated_criteria_scores.append(criterion_score1) key1_scored_object = criterion_score1.convert_to_scored_object() criterion_score2 = next((criterion_score for criterion_score in criteria_scores if criterion_score.answer_id == answer2_id and criterion_score.criterion_id == criterion_id), AnswerCriterionScore(assignment_id=assignment.id, answer_id=answer2_id, criterion_id=criterion_id) ) updated_criteria_scores.append(criterion_score2) key2_scored_object = criterion_score2.convert_to_scored_object() criterion_result_1, criterion_result_2 = calculate_score_1vs1( package_name=assignment.scoring_algorithm.value, key1_scored_object=key1_scored_object, key2_scored_object=key2_scored_object, winner=comparison_criterion.comparison_pair_winner(), other_comparison_pairs=[ cc.convert_to_comparison_pair() for cc in other_criterion_comparisons if cc.criterion_id == criterion_id ], log=current_app.logger ) for score, result in [(criterion_score1, criterion_result_1), (criterion_score2, criterion_result_2)]: score.score = result.score score.variable1 = result.variable1 score.variable2 = result.variable2 score.rounds = result.rounds score.wins = result.wins score.loses = result.loses score.opponents = result.opponents updated_scores = [] score1 = next((score for score in scores if score.answer_id == answer1_id), AnswerScore(assignment_id=assignment.id, answer_id=answer1_id) ) updated_scores.append(score1) key1_scored_object = score1.convert_to_scored_object() score2 = next((score for score in scores if score.answer_id == answer2_id), AnswerScore(assignment_id=assignment.id, answer_id=answer2_id) ) updated_scores.append(score2) key2_scored_object = score2.convert_to_scored_object() result_1, result_2 = calculate_score_1vs1( package_name=assignment.scoring_algorithm.value, key1_scored_object=key1_scored_object, key2_scored_object=key2_scored_object, winner=comparison.comparison_pair_winner(), other_comparison_pairs=[c.convert_to_comparison_pair() for c in other_comparisons], log=current_app.logger ) for score, result in [(score1, result_1), (score2, result_2)]: score.score = result.score score.variable1 = result.variable1 score.variable2 = result.variable2 score.rounds = result.rounds score.wins = result.wins score.loses = result.loses score.opponents = result.opponents db.session.add_all(updated_criteria_scores) db.session.add_all(updated_scores) db.session.commit() return updated_scores
def test_calculate_score_1vs1(self): self.comparisons = [ ComparisonPair(key1=1,key2=2, winner=ComparisonWinner.key1), ComparisonPair(key1=1,key2=3, winner=ComparisonWinner.key1) ] # test comparative judgement score algorithm self.package_name = "comparative_judgement" key1_scored_object = ScoredObject( key=2, score=None, variable1=None, variable2=None, rounds=None, wins=None, loses=None, opponents=None ) key2_scored_object = ScoredObject( key=3, score=None, variable1=None, variable2=None, rounds=None, wins=None, loses=None, opponents=None ) winner = ComparisonWinner.key1 key2_results, key3_results = calculate_score_1vs1( package_name=self.package_name, key1_scored_object=key1_scored_object, key2_scored_object=key2_scored_object, winner=winner, other_comparison_pairs=self.comparisons ) self.assertIsInstance(key2_results, ScoredObject) self.assertIsInstance(key2_results.score, float) self.assertIsInstance(key2_results.variable1, float) self.assertIsNone(key2_results.variable2) self.assertEqual(key2_results.rounds, 2) self.assertEqual(key2_results.opponents, 2) self.assertEqual(key2_results.wins, 1) self.assertEqual(key2_results.loses, 1) self.assertIsInstance(key3_results, ScoredObject) self.assertIsInstance(key3_results.score, float) self.assertIsInstance(key3_results.variable1, float) self.assertIsNone(key3_results.variable2) self.assertEqual(key3_results.rounds, 2) self.assertEqual(key3_results.opponents, 2) self.assertEqual(key3_results.wins, 0) self.assertEqual(key3_results.loses, 2) # test elo rating algorithm self.package_name = "elo_rating" key1_scored_object = ScoredObject( key=2, score=1395, variable1=1395, variable2=None, rounds=None, wins=None, loses=None, opponents=None ) key2_scored_object = ScoredObject( key=3, score=1396, variable1=1396, variable2=None, rounds=None, wins=None, loses=None, opponents=None ) winner = ComparisonWinner.key1 key2_results, key3_results = calculate_score_1vs1( package_name=self.package_name, key1_scored_object=key1_scored_object, key2_scored_object=key2_scored_object, winner=winner, other_comparison_pairs=self.comparisons ) self.assertIsInstance(key2_results, ScoredObject) self.assertIsInstance(key2_results.score, float) self.assertIsInstance(key2_results.variable1, float) self.assertIsNone(key2_results.variable2) self.assertEqual(key2_results.rounds, 2) self.assertEqual(key2_results.opponents, 2) self.assertEqual(key2_results.wins, 1) self.assertEqual(key2_results.loses, 1) self.assertIsInstance(key3_results, ScoredObject) self.assertIsInstance(key3_results.score, float) self.assertIsInstance(key3_results.variable1, float) self.assertIsNone(key3_results.variable2) self.assertEqual(key3_results.rounds, 2) self.assertEqual(key3_results.opponents, 2) self.assertEqual(key3_results.wins, 0) self.assertEqual(key3_results.loses, 2) # test true skill rating algorithm self.package_name = "true_skill_rating" key1_scored_object = ScoredObject( key=2, score=-0.909, variable1=20.604, variable2=7.171, rounds=None, wins=None, loses=None, opponents=None ) key2_scored_object = ScoredObject( key=3, score=-0.058, variable1=21.542, variable2=7.200, rounds=None, wins=None, loses=None, opponents=None ) winner = ComparisonWinner.key1 key2_results, key3_results = calculate_score_1vs1( package_name=self.package_name, key1_scored_object=key1_scored_object, key2_scored_object=key2_scored_object, winner=winner, other_comparison_pairs=self.comparisons ) self.assertIsInstance(key2_results, ScoredObject) self.assertIsInstance(key2_results.score, float) self.assertIsInstance(key2_results.variable1, float) self.assertIsInstance(key2_results.variable2, float) self.assertEqual(key2_results.rounds, 2) self.assertEqual(key2_results.opponents, 2) self.assertEqual(key2_results.wins, 1) self.assertEqual(key2_results.loses, 1) self.assertIsInstance(key3_results, ScoredObject) self.assertIsInstance(key3_results.score, float) self.assertIsInstance(key3_results.variable1, float) self.assertIsInstance(key3_results.variable2, float) self.assertEqual(key3_results.rounds, 2) self.assertEqual(key3_results.opponents, 2) self.assertEqual(key3_results.wins, 0) self.assertEqual(key3_results.loses, 2)