def test_generate_pair(self): # test adaptive pair algorithm self.package_name = "adaptive" results = generate_pair(package_name=self.package_name, scored_objects=self.scored_objects, comparison_pairs=self.comparisons) self.assertIsInstance(results, ComparisonPair) min_key = min([results.key1, results.key2]) max_key = max([results.key1, results.key2]) # round zero items should be selected self.assertEqual(min_key, 3) self.assertEqual(max_key, 4) self.assertEqual(results.winner, None) # test random pair algorithm self.package_name = "random" results = generate_pair(package_name=self.package_name, scored_objects=self.scored_objects, comparison_pairs=self.comparisons) self.assertIsInstance(results, ComparisonPair) min_key = min([results.key1, results.key2]) max_key = max([results.key1, results.key2]) # round zero items should be selected self.assertEqual(min_key, 3) self.assertEqual(max_key, 4) self.assertEqual(results.winner, None)
def test_generate_pair(self): # test adaptive pair algorithm self.package_name = "adaptive" results = generate_pair( package_name=self.package_name, scored_objects=self.scored_objects, comparison_pairs=self.comparisons ) self.assertIsInstance(results, ComparisonPair) min_key = min([results.key1, results.key2]) max_key = max([results.key1, results.key2]) # round zero items should be selected self.assertEqual(min_key, 3) self.assertEqual(max_key, 4) self.assertEqual(results.winner, None) # test adaptive min delta pair algorithm self.package_name = "adaptive_min_delta" results = generate_pair( package_name=self.package_name, scored_objects=self.scored_objects, comparison_pairs=self.comparisons, criterion_scores=self.criterion_scores, criterion_weights=self.criterion_weights ) self.assertIsInstance(results, ComparisonPair) min_key = min([results.key1, results.key2]) max_key = max([results.key1, results.key2]) # round zero items should be selected self.assertEqual(min_key, 3) self.assertEqual(max_key, 4) self.assertEqual(results.winner, None) # test random pair algorithm self.package_name = "random" results = generate_pair( package_name=self.package_name, scored_objects=self.scored_objects, comparison_pairs=self.comparisons ) self.assertIsInstance(results, ComparisonPair) min_key = min([results.key1, results.key2]) max_key = max([results.key1, results.key2]) # round zero items should be selected self.assertEqual(min_key, 3) self.assertEqual(max_key, 4) self.assertEqual(results.winner, None)
def _get_new_comparison_pair(cls, course_id, assignment_id, user_id, pairing_algorithm, comparisons): from . import Assignment, UserCourse, CourseRole, Answer, AnswerScore, PairingAlgorithm # ineligible authors - eg. instructors, TAs, dropped student, current user non_students = UserCourse.query \ .filter(and_( UserCourse.course_id == course_id, UserCourse.course_role != CourseRole.student )) ineligible_user_ids = [non_student.user_id \ for non_student in non_students] ineligible_user_ids.append(user_id) answers_with_score = Answer.query \ .with_entities(Answer, AnswerScore.score ) \ .outerjoin(AnswerScore) \ .filter(and_( Answer.user_id.notin_(ineligible_user_ids), Answer.assignment_id == assignment_id, Answer.active == True, Answer.practice == False, Answer.draft == False )) \ .all() scored_objects = [] for answer_with_score in answers_with_score: scored_objects.append( ScoredObject(key=answer_with_score.Answer.id, score=answer_with_score.score, rounds=answer_with_score.Answer.round, variable1=None, variable2=None, wins=None, loses=None, opponents=None)) comparison_pairs = [ comparison.convert_to_comparison_pair() for comparison in comparisons ] comparison_pair = generate_pair(package_name=pairing_algorithm.value, scored_objects=scored_objects, comparison_pairs=comparison_pairs, log=current_app.logger) return comparison_pair
comparisons = [] for round in range(1, NUMBER_OF_ROUNDS + 1): if len(students) == 0: break for comparison_in_round in range(ROUND_LENGTH): if len(students) == 0: break random.shuffle(students) student = students[0] student_comparisons = student['comparisons_completed'] comparison_pair = generate_pair( package_name=pairing_package_name, scored_objects=answers, comparison_pairs=student_comparisons) key1 = comparison_pair.key1 key2 = comparison_pair.key2 winner = select_winner(round, student['key'], key1, key2) comparison_pair = comparison_pair._replace(winner=winner) comparisons.append(comparison_pair) student['comparisons_completed'].append(comparison_pair) student[ 'comparisons_left'] = student['comparisons_left'] - 1 if student['comparisons_left'] <= 0: students.remove(student) index1 = next(index for index, answer in enumerate(answers) if answer.key == key1)
def _run(file_path, pairing_package_name, scoring_package_name, winner_selector, correct_rate): random.seed() numpy.random.seed() actual_grades = numpy.random.normal(0.78, 0.1, NUMBER_OF_ANSWERS) grade_by_answer_key = {} answers = [] results = [] for key, grade in enumerate(actual_grades): grade_by_answer_key[key+1] = grade answers.append(ScoredObject( key=key+1, score=0, variable1=None, variable2=None, rounds=0, opponents=0, wins=0, loses=0 )) students = [] for key in range(NUMBER_OF_STUDENTS): students.append({ 'key': key, 'comparisons_left': NUMBER_OF_COMPARISONS_PER_STUDENT, 'comparisons_completed': [] }) comparisons = [] for round_count in range(1, NUMBER_OF_ROUNDS+1): if len(students) == 0: break for comparison_in_round in range(ROUND_LENGTH): if len(students) == 0: break student = random.choice(students) student_comparisons = student['comparisons_completed'] comparison_pair = generate_pair( package_name=pairing_package_name, scored_objects=answers, comparison_pairs=student_comparisons ) key1 = comparison_pair.key1 key1_grade = grade_by_answer_key[key1] key2 = comparison_pair.key2 key2_grade = grade_by_answer_key[key2] winner = _decide_winner( winner_selector, correct_rate, key1_grade, key2_grade ) comparison_pair = comparison_pair._replace(winner=winner) comparisons.append(comparison_pair) student['comparisons_completed'].append(comparison_pair) student['comparisons_left'] -= 1 if student['comparisons_left'] <= 0: indexes = [i for i, s in enumerate(students) if student['key'] == s['key']] del students[indexes[0]] index1 = next(index for index, answer in enumerate(answers) if answer.key == key1) index2 = next(index for index, answer in enumerate(answers) if answer.key == key2) result1, results2 = calculate_score_1vs1( package_name=scoring_package_name, key1_scored_object=answers[index1], key2_scored_object=answers[index2], winner=winner, other_comparison_pairs=comparisons ) answers[index1] = result1 answers[index2] = results2 current_scores = [answer.score for answer in answers] r_value, pearsonr_p_value = pearsonr(actual_grades, current_scores) results.append(str(r_value)) #print("Round {} ----------- pearsonr={} value=={}".format( # round_count, r_value, pearsonr_p_value #)) with open(file_path, "a") as csvfile: out = csv.writer(csvfile) out.writerow(results)
def _get_new_comparison_pair(cls, course_id, assignment_id, user_id, group_id, pairing_algorithm, comparisons): from . import Assignment, UserCourse, CourseRole, Answer, AnswerScore, \ PairingAlgorithm, AnswerCriterionScore, AssignmentCriterion, Group # exclude current user and those without a proper role. # note that sys admin (not enrolled in the course and thus no course role) can create answers. # they are considered eligible ineligibles = UserCourse.query \ .with_entities(UserCourse.user_id) \ .filter(and_( UserCourse.course_id == course_id, UserCourse.course_role == CourseRole.dropped )) \ .all() ineligible_user_ids = [ ineligible.user_id for ineligible in ineligibles ] ineligible_user_ids.append(user_id) query = Answer.query \ .with_entities(Answer, AnswerScore.score) \ .outerjoin(AnswerScore, AnswerScore.answer_id == Answer.id) \ .filter(and_( or_( ~Answer.user_id.in_(ineligible_user_ids), Answer.user_id == None # don't filter out group answers ), Answer.assignment_id == assignment_id, Answer.active == True, Answer.practice == False, Answer.draft == False, Answer.comparable == True )) if group_id: query = query.filter(Answer.group_id != group_id) answers_with_score = query.all() scored_objects = [] for answer_with_score in answers_with_score: scored_objects.append( ScoredObject(key=answer_with_score.Answer.id, score=answer_with_score.score, rounds=answer_with_score.Answer.round, variable1=None, variable2=None, wins=None, loses=None, opponents=None)) comparison_pairs = [ comparison.convert_to_comparison_pair() for comparison in comparisons ] # adaptive min delta algo requires extra criterion specific parameters if pairing_algorithm == PairingAlgorithm.adaptive_min_delta: # retrieve extra criterion score data answer_criterion_scores = AnswerCriterionScore.query \ .with_entities(AnswerCriterionScore.answer_id, AnswerCriterionScore.criterion_id, AnswerCriterionScore.score) \ .join(Answer) \ .filter(and_( Answer.user_id.notin_(ineligible_user_ids), Answer.assignment_id == assignment_id, Answer.active == True, Answer.practice == False, Answer.draft == False )) \ .all() assignment_criterion_weights = AssignmentCriterion.query \ .with_entities(AssignmentCriterion.criterion_id, AssignmentCriterion.weight) \ .filter(and_( AssignmentCriterion.assignment_id == assignment_id, AssignmentCriterion.active == True )) \ .all() criterion_scores = {} for criterion_score in answer_criterion_scores: scores = criterion_scores.setdefault(criterion_score.answer_id, {}) scores[criterion_score.criterion_id] = criterion_score.score criterion_weights = {} for the_weight in assignment_criterion_weights: criterion_weights[the_weight.criterion_id] = \ the_weight.weight comparison_pair = generate_pair( package_name=pairing_algorithm.value, scored_objects=scored_objects, comparison_pairs=comparison_pairs, criterion_scores=criterion_scores, criterion_weights=criterion_weights, log=current_app.logger) else: comparison_pair = generate_pair( package_name=pairing_algorithm.value, scored_objects=scored_objects, comparison_pairs=comparison_pairs, log=current_app.logger) return comparison_pair
def _run(file_path, pairing_package_name, scoring_package_name, winner_selector, correct_rate, actual_grades, repetition_count): random.seed() numpy.random.seed() while repetition_count < REPETITIONS: grade_by_answer_key = {} answers = [] results = [] for key, grade in enumerate(actual_grades): grade_by_answer_key[key + 1] = grade answers.append( ScoredObject(key=key + 1, score=0, variable1=None, variable2=None, rounds=0, opponents=0, wins=0, loses=0)) students = [] for key in range(NUMBER_OF_STUDENTS): students.append({ 'key': key, 'comparisons_left': NUMBER_OF_COMPARISONS_PER_STUDENT, 'comparisons_completed': [] }) comparisons = [] for round_count in range(1, NUMBER_OF_ROUNDS + 1): if len(students) == 0: break for comparison_in_round in range(ROUND_LENGTH): if len(students) == 0: break student = random.choice(students) student_comparisons = student['comparisons_completed'] comparison_pair = generate_pair( package_name=pairing_package_name, scored_objects=answers, comparison_pairs=student_comparisons) key1 = comparison_pair.key1 key1_grade = grade_by_answer_key[key1] key2 = comparison_pair.key2 key2_grade = grade_by_answer_key[key2] winner = _decide_winner(winner_selector, correct_rate, key1_grade, key2_grade) comparison_pair = comparison_pair._replace(winner=winner) comparisons.append(comparison_pair) student['comparisons_completed'].append(comparison_pair) student['comparisons_left'] -= 1 if student['comparisons_left'] <= 0: indexes = [ i for i, s in enumerate(students) if student['key'] == s['key'] ] del students[indexes[0]] index1 = next(index for index, answer in enumerate(answers) if answer.key == key1) index2 = next(index for index, answer in enumerate(answers) if answer.key == key2) result1, results2 = calculate_score_1vs1( package_name=scoring_package_name, key1_scored_object=answers[index1], key2_scored_object=answers[index2], winner=winner, other_comparison_pairs=comparisons) answers[index1] = result1 answers[index2] = results2 current_scores = [answer.score for answer in answers] r_value, pearsonr_p_value = pearsonr(ACTUAL_GRADES, current_scores) results.append(str(r_value)) #print("Round {} ----------- pearsonr={} value=={}".format( # round_count, r_value, pearsonr_p_value #)) with open(file_path, "a") as csvfile: out = csv.writer(csvfile) out.writerow(results) # prepare for next run repetition_count += 1 actual_grades = [answer.score for answer in answers]
for round in range(1, NUMBER_OF_ROUNDS+1): if len(students) == 0: break for comparison_in_round in range(ROUND_LENGTH): if len(students) == 0: break random.shuffle(students) student = students[0] student_comparisons = student['comparisons_completed'] comparison_pair = generate_pair( package_name=pairing_package_name, scored_objects=answers, comparison_pairs=student_comparisons ) key1 = comparison_pair.key1 key2 = comparison_pair.key2 winning_key = select_winner(round, student['key'], key1, key2) comparison_pair = comparison_pair._replace(winning_key=winning_key) comparisons.append(comparison_pair) student['comparisons_completed'].append(comparison_pair) student['comparisons_left'] = student['comparisons_left'] - 1 if student['comparisons_left'] <= 0: students.remove(student) index1 = next(index for index, answer in enumerate(answers) if answer.key == key1) index2 = next(index for index, answer in enumerate(answers) if answer.key == key2)
def _get_new_comparison_pair(cls, course_id, assignment_id, user_id, group_id, pairing_algorithm, comparisons): from . import Assignment, UserCourse, CourseRole, Answer, AnswerScore, \ PairingAlgorithm, AnswerCriterionScore, AssignmentCriterion, Group # exclude current user and those without a proper role. # note that sys admin (not enrolled in the course and thus no course role) can create answers. # they are considered eligible ineligibles = UserCourse.query \ .with_entities(UserCourse.user_id) \ .filter(and_( UserCourse.course_id == course_id, UserCourse.course_role == CourseRole.dropped )) \ .all() ineligible_user_ids = [ineligible.user_id for ineligible in ineligibles] ineligible_user_ids.append(user_id) query = Answer.query \ .with_entities(Answer, AnswerScore.score) \ .outerjoin(AnswerScore, AnswerScore.answer_id == Answer.id) \ .filter(and_( or_( ~Answer.user_id.in_(ineligible_user_ids), Answer.user_id == None # don't filter out group answers ), Answer.assignment_id == assignment_id, Answer.active == True, Answer.practice == False, Answer.draft == False, Answer.comparable == True )) if group_id: query = query.filter(Answer.group_id != group_id) answers_with_score = query.all() scored_objects = [] for answer_with_score in answers_with_score: scored_objects.append(ScoredObject( key=answer_with_score.Answer.id, score=answer_with_score.score, rounds=answer_with_score.Answer.round, variable1=None, variable2=None, wins=None, loses=None, opponents=None )) comparison_pairs = [comparison.convert_to_comparison_pair() for comparison in comparisons] # adaptive min delta algo requires extra criterion specific parameters if pairing_algorithm == PairingAlgorithm.adaptive_min_delta: # retrieve extra criterion score data answer_criterion_scores = AnswerCriterionScore.query \ .with_entities(AnswerCriterionScore.answer_id, AnswerCriterionScore.criterion_id, AnswerCriterionScore.score) \ .join(Answer) \ .filter(and_( Answer.user_id.notin_(ineligible_user_ids), Answer.assignment_id == assignment_id, Answer.active == True, Answer.practice == False, Answer.draft == False )) \ .all() assignment_criterion_weights = AssignmentCriterion.query \ .with_entities(AssignmentCriterion.criterion_id, AssignmentCriterion.weight) \ .filter(and_( AssignmentCriterion.assignment_id == assignment_id, AssignmentCriterion.active == True )) \ .all() criterion_scores = {} for criterion_score in answer_criterion_scores: scores = criterion_scores.setdefault(criterion_score.answer_id, {}) scores[criterion_score.criterion_id] = criterion_score.score criterion_weights = {} for the_weight in assignment_criterion_weights: criterion_weights[the_weight.criterion_id] = \ the_weight.weight comparison_pair = generate_pair( package_name=pairing_algorithm.value, scored_objects=scored_objects, comparison_pairs=comparison_pairs, criterion_scores=criterion_scores, criterion_weights=criterion_weights, log=current_app.logger ) else: comparison_pair = generate_pair( package_name=pairing_algorithm.value, scored_objects=scored_objects, comparison_pairs=comparison_pairs, log=current_app.logger ) return comparison_pair